failproofai 1.0.1 → 1.0.2-beta.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.next/standalone/.next/BUILD_ID +1 -1
- package/.next/standalone/.next/build-manifest.json +3 -3
- package/.next/standalone/.next/prerender-manifest.json +6 -6
- package/.next/standalone/.next/required-server-files.json +1 -1
- package/.next/standalone/.next/server/app/_global-error/page/server-reference-manifest.json +1 -1
- package/.next/standalone/.next/server/app/_global-error/page.js +2 -2
- package/.next/standalone/.next/server/app/_global-error/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/_global-error/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/_global-error.html +1 -1
- package/.next/standalone/.next/server/app/_global-error.rsc +7 -7
- package/.next/standalone/.next/server/app/_global-error.segments/__PAGE__.segment.rsc +6 -6
- package/.next/standalone/.next/server/app/_global-error.segments/_full.segment.rsc +7 -7
- package/.next/standalone/.next/server/app/_global-error.segments/_tree.segment.rsc +1 -1
- package/.next/standalone/.next/server/app/_not-found/page/server-reference-manifest.json +1 -1
- package/.next/standalone/.next/server/app/_not-found/page.js +2 -2
- package/.next/standalone/.next/server/app/_not-found/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/_not-found/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/_not-found.html +1 -1
- package/.next/standalone/.next/server/app/_not-found.rsc +15 -15
- package/.next/standalone/.next/server/app/_not-found.segments/_full.segment.rsc +15 -15
- package/.next/standalone/.next/server/app/_not-found.segments/_not-found/__PAGE__.segment.rsc +14 -14
- package/.next/standalone/.next/server/app/_not-found.segments/_tree.segment.rsc +2 -2
- package/.next/standalone/.next/server/app/api/audit/invite/route.js +3 -2
- package/.next/standalone/.next/server/app/api/audit/invite/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/audit/run/route.js +5 -4
- package/.next/standalone/.next/server/app/api/audit/run/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/audit/status/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/auth/login-request/route.js +2 -2
- package/.next/standalone/.next/server/app/api/auth/login-request/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/auth/login-verify/route.js +4 -3
- package/.next/standalone/.next/server/app/api/auth/login-verify/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/auth/logout/route.js +4 -4
- package/.next/standalone/.next/server/app/api/auth/logout/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/auth/status/route.js +4 -4
- package/.next/standalone/.next/server/app/api/auth/status/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/api/download/[project]/[session]/route.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/audit/page/server-reference-manifest.json +2 -2
- package/.next/standalone/.next/server/app/audit/page.js +2 -2
- package/.next/standalone/.next/server/app/audit/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/audit/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/index.html +1 -1
- package/.next/standalone/.next/server/app/index.rsc +15 -15
- package/.next/standalone/.next/server/app/index.segments/__PAGE__.segment.rsc +14 -14
- package/.next/standalone/.next/server/app/index.segments/_full.segment.rsc +15 -15
- package/.next/standalone/.next/server/app/index.segments/_tree.segment.rsc +2 -2
- package/.next/standalone/.next/server/app/page/server-reference-manifest.json +1 -1
- package/.next/standalone/.next/server/app/page.js +2 -2
- package/.next/standalone/.next/server/app/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/policies/page/server-reference-manifest.json +66 -22
- package/.next/standalone/.next/server/app/policies/page.js +5 -4
- package/.next/standalone/.next/server/app/policies/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/policies/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/project/[name]/page/server-reference-manifest.json +1 -1
- package/.next/standalone/.next/server/app/project/[name]/page.js +2 -2
- package/.next/standalone/.next/server/app/project/[name]/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/project/[name]/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/project/[name]/session/[sessionId]/page/react-loadable-manifest.json +2 -2
- package/.next/standalone/.next/server/app/project/[name]/session/[sessionId]/page/server-reference-manifest.json +2 -2
- package/.next/standalone/.next/server/app/project/[name]/session/[sessionId]/page.js +2 -2
- package/.next/standalone/.next/server/app/project/[name]/session/[sessionId]/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/project/[name]/session/[sessionId]/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/projects/page/server-reference-manifest.json +1 -1
- package/.next/standalone/.next/server/app/projects/page.js +2 -2
- package/.next/standalone/.next/server/app/projects/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/projects/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/app/settings/page/server-reference-manifest.json +4 -4
- package/.next/standalone/.next/server/app/settings/page.js +1 -1
- package/.next/standalone/.next/server/app/settings/page.js.nft.json +1 -1
- package/.next/standalone/.next/server/app/settings/page_client-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/chunks/{[externals]__1qzqcio._.js → [externals]__0e32-7c._.js} +1 -1
- package/.next/standalone/.next/server/chunks/[externals]__1s462yl._.js +3 -0
- package/.next/standalone/.next/server/chunks/[root-of-the-server]__01eqny5._.js +3 -0
- package/.next/standalone/.next/server/chunks/[root-of-the-server]__0_0xu5z._.js +1 -1
- package/.next/standalone/.next/server/chunks/{[root-of-the-server]__1p8poxd._.js → [root-of-the-server]__0igxt32._.js} +2 -2
- package/.next/standalone/.next/server/chunks/[root-of-the-server]__0neqlu-._.js +3 -0
- package/.next/standalone/.next/server/chunks/[root-of-the-server]__0o07qi9._.js +3 -0
- package/.next/standalone/.next/server/chunks/[root-of-the-server]__0t9oqnn._.js +3 -0
- package/.next/standalone/.next/server/chunks/_0bs-v8p._.js +3 -0
- package/.next/standalone/.next/server/chunks/_0tovk6q._.js +3 -0
- package/.next/standalone/.next/server/chunks/_0trp3yc._.js +3 -0
- package/.next/standalone/.next/server/chunks/_185gyk9._.js +3 -0
- package/.next/standalone/.next/server/chunks/_1ek68ln._.js +5 -6
- package/.next/standalone/.next/server/chunks/lib_telemetry-id_ts_1k8gj9o._.js +3 -0
- package/.next/standalone/.next/server/chunks/{node_modules_posthog-node_dist_entrypoints_index_node_mjs_09d5c79._.js → node_modules_posthog-node_dist_entrypoints_index_node_mjs_04op_ou._.js} +1 -1
- package/.next/standalone/.next/server/chunks/{node_modules_posthog-node_dist_entrypoints_index_node_mjs_1d2g73o._.js → node_modules_posthog-node_dist_entrypoints_index_node_mjs_0dr20uz._.js} +1 -1
- package/.next/standalone/.next/server/chunks/package_json_[json]_cjs_1nxcc4v._.js +1 -1
- package/.next/standalone/.next/server/chunks/src_hooks_0xbw3bu._.js +12 -0
- package/.next/standalone/.next/server/chunks/src_hooks_fp-home_ts_09kv0bn._.js +1 -1
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__013jr2b._.js +4 -0
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__01wy8d-._.js +4 -0
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__02npjtd._.js +4 -0
- package/.next/standalone/.next/server/chunks/ssr/{[root-of-the-server]__0spkm68._.js → [root-of-the-server]__07gm3zl._.js} +21 -20
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__0da85px._.js +4 -0
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__0ftmoxc._.js +4 -0
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__0oa1lav._.js +1 -1
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__0p-5p8u._.js +4 -0
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__0s740oi._.js +6 -0
- package/.next/standalone/.next/server/chunks/ssr/{[root-of-the-server]__07tbbf3._.js → [root-of-the-server]__0zig5sv._.js} +2 -2
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__192nhvh._.js +5 -0
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__1p2otjt._.js +4 -0
- package/.next/standalone/.next/server/chunks/ssr/{_0lvyuv4._.js → _00e9tnf._.js} +2 -2
- package/.next/standalone/.next/server/chunks/ssr/_08x1r5t._.js +1 -1
- package/.next/standalone/.next/server/chunks/ssr/{_0u5zauv._.js → _12l4a_k._.js} +1 -1
- package/.next/standalone/.next/server/chunks/ssr/_1u8-lu2._.js +1 -1
- package/.next/standalone/.next/server/chunks/ssr/_1zopuov._.js +1 -1
- package/.next/standalone/.next/server/chunks/ssr/app_audit__components_audit-dashboard_tsx_0p9ud47._.js +13 -13
- package/.next/standalone/.next/server/chunks/ssr/app_global-error_tsx_1kp6l3x._.js +1 -1
- package/.next/standalone/.next/server/chunks/ssr/app_policies_hooks-client_tsx_19dqvpc._.js +2 -2
- package/.next/standalone/.next/server/chunks/ssr/app_settings_settings-client_tsx_20lq-mq._.js +1 -1
- package/.next/standalone/.next/server/chunks/ssr/src_hooks_08o7bqo._.js +10 -0
- package/.next/standalone/.next/server/chunks/ssr/src_hooks_builtin-policies_ts_09j2ndl._.js +1 -1
- package/.next/standalone/.next/server/chunks/ssr/src_hooks_fp-home_ts_0je3xkv._.js +1 -1
- package/.next/standalone/.next/server/middleware-build-manifest.js +3 -3
- package/.next/standalone/.next/server/middleware.js +1 -1
- package/.next/standalone/.next/server/pages/404.html +1 -1
- package/.next/standalone/.next/server/pages/500.html +1 -1
- package/.next/standalone/.next/server/server-reference-manifest.js +1 -1
- package/.next/standalone/.next/server/server-reference-manifest.json +71 -27
- package/.next/standalone/.next/static/chunks/{37fcpc36x9gpe.js → 0eqky-7n7dgdm.js} +1 -1
- package/.next/standalone/.next/static/chunks/{1ide1wl2f7zi1.js → 0sa9v5mhc8861.js} +13 -13
- package/.next/standalone/.next/static/chunks/1mfzsja6ftim5.css +2 -0
- package/.next/standalone/.next/static/chunks/{3k7mxbp9hiom0.js → 1nuqytbta44af.js} +1 -1
- package/.next/standalone/.next/static/chunks/1sl2xmn9inx3y.js +6 -0
- package/.next/standalone/.next/static/chunks/{2_2lhuojq2xop.js → 2aawsjld9ied_.js} +1 -1
- package/.next/standalone/.next/static/chunks/2gn7ut4svofm2.js +1 -0
- package/.next/standalone/.next/static/chunks/{31s1smp-g7gks.js → 2hs7ycvsav7v2.js} +1 -1
- package/.next/standalone/.next/static/chunks/{0k5050ezjvv49.js → 2i2yt3hzjredm.js} +1 -1
- package/.next/standalone/.next/static/chunks/{3c808qrfcq0im.js → 35xg6retpe6rp.js} +1 -1
- package/.next/standalone/.next/static/chunks/{2zpp1we2jv6tb.js → 3y54nw0i7dsjl.js} +1 -1
- package/.next/standalone/SECURITY.md +5 -4
- package/.next/standalone/app/actions/get-hooks-config.ts +108 -17
- package/.next/standalone/app/actions/pack-actions.ts +153 -0
- package/.next/standalone/app/audit/_components/run-progress.tsx +1 -1
- package/.next/standalone/app/audit/_components/share-templates.ts +13 -13
- package/.next/standalone/app/policies/hooks-client.tsx +320 -14
- package/.next/standalone/components/navbar.tsx +2 -28
- package/.next/standalone/components/reach-developers.tsx +8 -1
- package/.next/standalone/docs-old/agenteye/python-sdk-skill.mdx +1 -1
- package/.next/standalone/docs-old/agenteye/python-sdk.mdx +59 -35
- package/.next/standalone/docs-old/i18n/README.ar.md +1 -0
- package/.next/standalone/docs-old/i18n/README.de.md +1 -0
- package/.next/standalone/docs-old/i18n/README.es.md +1 -0
- package/.next/standalone/docs-old/i18n/README.fr.md +1 -0
- package/.next/standalone/docs-old/i18n/README.he.md +1 -0
- package/.next/standalone/docs-old/i18n/README.hi.md +1 -0
- package/.next/standalone/docs-old/i18n/README.it.md +1 -0
- package/.next/standalone/docs-old/i18n/README.ja.md +1 -0
- package/.next/standalone/docs-old/i18n/README.ko.md +1 -0
- package/.next/standalone/docs-old/i18n/README.pt-br.md +1 -0
- package/.next/standalone/docs-old/i18n/README.ru.md +1 -0
- package/.next/standalone/docs-old/i18n/README.tr.md +1 -0
- package/.next/standalone/docs-old/i18n/README.vi.md +1 -0
- package/.next/standalone/docs-old/i18n/README.zh.md +1 -0
- package/.next/standalone/fp-cloud-cli/CHANGELOG.md +431 -0
- package/.next/standalone/fp-cloud-cli/LICENSE +42 -0
- package/.next/standalone/fp-cloud-cli/MANIFEST.in +10 -0
- package/.next/standalone/fp-cloud-cli/README.md +259 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/__init__.py +10 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/__main__.py +4 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/_click_compat.py +64 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/_context.py +332 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/_version.py +1 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/analytics.py +432 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/analytics_config.py +77 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/analytics_registry.py +83 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/app.py +492 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/auth.py +160 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/client.py +1694 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/__init__.py +0 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/_write.py +214 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/agent_cmds.py +407 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/alerts_cmds.py +445 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/audits_cmds.py +1054 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/auth_cmds.py +512 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/errors_cmds.py +190 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/evals_cmds.py +161 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/events_cmds.py +159 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/fleet_cmds.py +416 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/guardrails_cmds.py +148 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/incidents_cmds.py +472 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/keys_cmds.py +407 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/list_cmds.py +63 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/orgs_cmds.py +319 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/policies_cmds.py +499 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/queries_cmds.py +378 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/sessions_cmds.py +151 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/settings_cmds.py +150 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/usage_cmds.py +35 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/commands/users_cmds.py +404 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/config.py +330 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/dates.py +78 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/enforcement.py +345 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/errors.py +98 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/models.py +891 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/orgs.py +30 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/output.py +6593 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/permissions.py +208 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/policy_check.py +290 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/py.typed +0 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/select.py +322 -0
- package/.next/standalone/fp-cloud-cli/fp_cli/theme.py +53 -0
- package/.next/standalone/fp-cloud-cli/pyproject.toml +106 -0
- package/.next/standalone/fp-cloud-cli/skill/SKILL.md +302 -0
- package/.next/standalone/fp-cloud-cli/skill/agents/openai.yaml +8 -0
- package/.next/standalone/fp-cloud-cli/skill/references/commands.md +254 -0
- package/.next/standalone/fp-cloud-cli/tests/__init__.py +0 -0
- package/.next/standalone/fp-cloud-cli/tests/conftest.py +92 -0
- package/.next/standalone/fp-cloud-cli/tests/test_alerting.py +644 -0
- package/.next/standalone/fp-cloud-cli/tests/test_analytics.py +385 -0
- package/.next/standalone/fp-cloud-cli/tests/test_audits.py +950 -0
- package/.next/standalone/fp-cloud-cli/tests/test_auth.py +98 -0
- package/.next/standalone/fp-cloud-cli/tests/test_auth_mode.py +272 -0
- package/.next/standalone/fp-cloud-cli/tests/test_click_compat.py +118 -0
- package/.next/standalone/fp-cloud-cli/tests/test_client.py +299 -0
- package/.next/standalone/fp-cloud-cli/tests/test_commands.py +814 -0
- package/.next/standalone/fp-cloud-cli/tests/test_config.py +99 -0
- package/.next/standalone/fp-cloud-cli/tests/test_dashboards_agent.py +340 -0
- package/.next/standalone/fp-cloud-cli/tests/test_dates.py +76 -0
- package/.next/standalone/fp-cloud-cli/tests/test_enforcement_logic.py +481 -0
- package/.next/standalone/fp-cloud-cli/tests/test_facets.py +170 -0
- package/.next/standalone/fp-cloud-cli/tests/test_failproofai_home.py +722 -0
- package/.next/standalone/fp-cloud-cli/tests/test_fp_home_contract.py +198 -0
- package/.next/standalone/fp-cloud-cli/tests/test_hardening.py +186 -0
- package/.next/standalone/fp-cloud-cli/tests/test_help_table_coverage.py +127 -0
- package/.next/standalone/fp-cloud-cli/tests/test_keys_queries.py +496 -0
- package/.next/standalone/fp-cloud-cli/tests/test_list.py +59 -0
- package/.next/standalone/fp-cloud-cli/tests/test_multivalue.py +183 -0
- package/.next/standalone/fp-cloud-cli/tests/test_no_customer_identifiers.py +168 -0
- package/.next/standalone/fp-cloud-cli/tests/test_operator.py +492 -0
- package/.next/standalone/fp-cloud-cli/tests/test_orgs.py +692 -0
- package/.next/standalone/fp-cloud-cli/tests/test_output.py +1732 -0
- package/.next/standalone/fp-cloud-cli/tests/test_policy_check.py +191 -0
- package/.next/standalone/fp-cloud-cli/tests/test_readme_matches_reality.py +125 -0
- package/.next/standalone/fp-cloud-cli/tests/test_review_fixes.py +161 -0
- package/.next/standalone/fp-cloud-cli/tests/test_telemetry_completeness.py +90 -0
- package/.next/standalone/fp-cloud-cli/tests/test_usage.py +115 -0
- package/.next/standalone/fp-cloud-cli/tests/test_v1_origin_diagnostic.py +70 -0
- package/.next/standalone/fp-cloud-cli/tests/test_v1_routing.py +214 -0
- package/.next/standalone/fp-cloud-cli/tests/test_whoami.py +104 -0
- package/.next/standalone/fp-cloud-cli/uv.lock +552 -0
- package/.next/standalone/package.json +29 -22
- package/.next/standalone/sdk/python/CHANGELOG.md +70 -0
- package/.next/standalone/sdk/python/LICENSE +42 -0
- package/.next/standalone/sdk/python/MANIFEST.in +21 -0
- package/.next/standalone/sdk/python/README.md +466 -0
- package/.next/standalone/sdk/python/docs/README.md +249 -0
- package/.next/standalone/sdk/python/docs/_shared/README.md +43 -0
- package/.next/standalone/sdk/python/docs/_shared/__init__.py +195 -0
- package/.next/standalone/sdk/python/docs/crewai/README.md +328 -0
- package/.next/standalone/sdk/python/docs/crewai/examples/quickstart.py +61 -0
- package/.next/standalone/sdk/python/docs/crewai/examples/research_crew.py +92 -0
- package/.next/standalone/sdk/python/docs/langgraph/README.md +481 -0
- package/.next/standalone/sdk/python/docs/langgraph/examples/quickstart.py +51 -0
- package/.next/standalone/sdk/python/docs/langgraph/examples/research_agent.py +86 -0
- package/.next/standalone/sdk/python/docs/langgraph/examples/supervisor_handoff.py +83 -0
- package/.next/standalone/sdk/python/docs/llama_index/README.md +332 -0
- package/.next/standalone/sdk/python/docs/llama_index/examples/quickstart.py +62 -0
- package/.next/standalone/sdk/python/docs/llama_index/examples/research_agent.py +93 -0
- package/.next/standalone/sdk/python/docs/manual/README.md +491 -0
- package/.next/standalone/sdk/python/docs/manual/examples/quickstart.py +57 -0
- package/.next/standalone/sdk/python/docs/manual/examples/research_agent.py +133 -0
- package/.next/standalone/sdk/python/docs/pydantic_ai/README.md +323 -0
- package/.next/standalone/sdk/python/docs/pydantic_ai/examples/quickstart.py +48 -0
- package/.next/standalone/sdk/python/docs/pydantic_ai/examples/research_agent.py +94 -0
- package/.next/standalone/sdk/python/failproofai_sdk/__init__.py +131 -0
- package/.next/standalone/sdk/python/failproofai_sdk/_context.py +218 -0
- package/.next/standalone/sdk/python/failproofai_sdk/_environment.py +78 -0
- package/.next/standalone/sdk/python/failproofai_sdk/_events.py +752 -0
- package/.next/standalone/sdk/python/failproofai_sdk/_resolver.py +118 -0
- package/.next/standalone/sdk/python/failproofai_sdk/_runtime.py +21 -0
- package/.next/standalone/sdk/python/failproofai_sdk/_schema.py +317 -0
- package/.next/standalone/sdk/python/failproofai_sdk/_scopes.py +422 -0
- package/.next/standalone/sdk/python/failproofai_sdk/_version.py +1 -0
- package/.next/standalone/sdk/python/failproofai_sdk/_writer.py +750 -0
- package/.next/standalone/sdk/python/failproofai_sdk/integrations/__init__.py +274 -0
- package/.next/standalone/sdk/python/failproofai_sdk/integrations/_compat.py +247 -0
- package/.next/standalone/sdk/python/failproofai_sdk/integrations/_core.py +1076 -0
- package/.next/standalone/sdk/python/failproofai_sdk/integrations/crewai.py +1669 -0
- package/.next/standalone/sdk/python/failproofai_sdk/integrations/langchain.py +2151 -0
- package/.next/standalone/sdk/python/failproofai_sdk/integrations/llama_index.py +1588 -0
- package/.next/standalone/sdk/python/failproofai_sdk/integrations/pydantic_ai.py +977 -0
- package/.next/standalone/sdk/python/failproofai_sdk/py.typed +0 -0
- package/.next/standalone/sdk/python/pyproject.toml +92 -0
- package/.next/standalone/sdk/python/skill/SKILL.md +399 -0
- package/.next/standalone/sdk/python/skill/agents/openai.yaml +8 -0
- package/.next/standalone/sdk/python/skill/references/events.md +234 -0
- package/.next/standalone/sdk/python/skill/references/frameworks.md +314 -0
- package/.next/standalone/sdk/python/skill/references/install.md +100 -0
- package/.next/standalone/sdk/python/skill/references/integration.md +352 -0
- package/.next/standalone/sdk/python/tests/__init__.py +0 -0
- package/.next/standalone/sdk/python/tests/conftest.py +169 -0
- package/.next/standalone/sdk/python/tests/integrations/__init__.py +6 -0
- package/.next/standalone/sdk/python/tests/integrations/test_crewai.py +1299 -0
- package/.next/standalone/sdk/python/tests/integrations/test_langchain.py +2338 -0
- package/.next/standalone/sdk/python/tests/integrations/test_llama_index.py +1576 -0
- package/.next/standalone/sdk/python/tests/integrations/test_pydantic_ai.py +1102 -0
- package/.next/standalone/sdk/python/tests/test_context.py +410 -0
- package/.next/standalone/sdk/python/tests/test_docs.py +411 -0
- package/.next/standalone/sdk/python/tests/test_durability.py +1352 -0
- package/.next/standalone/sdk/python/tests/test_encoding.py +571 -0
- package/.next/standalone/sdk/python/tests/test_integrations.py +1127 -0
- package/.next/standalone/sdk/python/tests/test_no_customer_identifiers.py +214 -0
- package/.next/standalone/sdk/python/tests/test_packaging.py +185 -0
- package/.next/standalone/sdk/python/tests/test_resolver_umbrella.py +178 -0
- package/.next/standalone/sdk/python/tests/test_scopes.py +487 -0
- package/.next/standalone/sdk/python/tests/test_sdk.py +923 -0
- package/.next/standalone/sdk/python/tests/test_server_contract.py +650 -0
- package/.next/standalone/sdk/python/tests/test_site_docs.py +371 -0
- package/.next/standalone/sdk/python/tests/test_skill_snippets.py +165 -0
- package/.next/standalone/sdk/python/tests/test_spool_contract.py +287 -0
- package/.next/standalone/sdk/python/tests/test_spool_creation.py +360 -0
- package/.next/standalone/sdk/python/tests/test_wire_format.py +279 -0
- package/.next/standalone/sdk/python/tests/test_zero_dependencies.py +378 -0
- package/.next/standalone/sdk/python/uv.lock +6005 -0
- package/.next/standalone/server.js +1 -1
- package/README.md +77 -21
- package/bin/failproofai.mjs +145 -11
- package/dist/cli.mjs +4482 -2205
- package/dist/index.js +5 -5
- package/dist/worker.mjs +2399 -1978
- package/package.json +29 -22
- package/policy-pack/SHA256SUMS +2 -0
- package/policy-pack/failproofai-pack.json +732 -0
- package/policy-pack/failproofai-pack.mjs +2017 -0
- package/scripts/build-policy-pack.mjs +121 -0
- package/scripts/changelog-section.py +77 -0
- package/scripts/prune-standalone.mjs +3 -0
- package/scripts/python-version.py +232 -0
- package/scripts/translate-docs/cli.ts +21 -0
- package/scripts/translate-docs/mintlify-nav.ts +85 -13
- package/scripts/translate-docs/translator.ts +66 -11
- package/src/audit/cache.ts +40 -3
- package/src/audit/cli.ts +1 -1
- package/src/audit/index.ts +1 -1
- package/src/audit/replay.ts +89 -3
- package/src/audit/report.ts +5 -5
- package/src/audit/schedule-cli.ts +49 -53
- package/src/hooks/builtin-policies.ts +217 -579
- package/src/hooks/cloud-enrollment-cli.ts +66 -27
- package/src/hooks/cloud-managed-policies.ts +6 -1
- package/src/hooks/configure-wizard.ts +1 -1
- package/src/hooks/custom-hooks-loader.ts +89 -6
- package/src/hooks/fp-home.ts +92 -4
- package/src/hooks/fp-reset.ts +26 -2
- package/src/hooks/handler.ts +181 -11
- package/src/hooks/harness-cli.ts +79 -25
- package/src/hooks/hook-activity-store.ts +13 -3
- package/src/hooks/install-prompt.ts +25 -3
- package/src/hooks/manager.ts +418 -172
- package/src/hooks/migrations.ts +2 -2
- package/src/hooks/pack-cli.ts +505 -0
- package/src/hooks/pack-failclosed.ts +189 -0
- package/src/hooks/pack-manifest.ts +344 -0
- package/src/hooks/pack-store.ts +930 -0
- package/src/hooks/policy-catalog.ts +535 -0
- package/src/hooks/policy-evaluator.ts +24 -16
- package/src/hooks/policy-presets.ts +4 -2
- package/src/hooks/policy-registry.ts +8 -2
- package/src/hooks/policy-types.ts +26 -0
- package/src/hooks/session-pause-cli.ts +33 -9
- package/src/hooks/tui.ts +570 -0
- package/src/hooks/types.ts +15 -1
- package/.next/standalone/.next/server/chunks/[root-of-the-server]__0_usx0j._.js +0 -3
- package/.next/standalone/.next/server/chunks/[root-of-the-server]__0_zk340._.js +0 -3
- package/.next/standalone/.next/server/chunks/[root-of-the-server]__0agdw2a._.js +0 -3
- package/.next/standalone/.next/server/chunks/[root-of-the-server]__0d1u_na._.js +0 -3
- package/.next/standalone/.next/server/chunks/[root-of-the-server]__0oy4kxb._.js +0 -3
- package/.next/standalone/.next/server/chunks/[root-of-the-server]__0zf034w._.js +0 -3
- package/.next/standalone/.next/server/chunks/[root-of-the-server]__1tvn4wf._.js +0 -3
- package/.next/standalone/.next/server/chunks/_0lxbzdq._.js +0 -3
- package/.next/standalone/.next/server/chunks/_1ger0a2._.js +0 -3
- package/.next/standalone/.next/server/chunks/_1zuiiy3._.js +0 -3
- package/.next/standalone/.next/server/chunks/node_modules_posthog-node_dist_entrypoints_index_node_mjs_1iiwm5e._.js +0 -3
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__0kc8_v5._.js +0 -4
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__0kgy9da._.js +0 -4
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__0mhqk5a._.js +0 -4
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__0pltpn8._.js +0 -4
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__0qkcyf-._.js +0 -4
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__0w0x_ok._.js +0 -4
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__1_dfocn._.js +0 -4
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__1bfex62._.js +0 -5
- package/.next/standalone/.next/server/chunks/ssr/[root-of-the-server]__1kzapg5._.js +0 -6
- package/.next/standalone/.next/static/chunks/36fw6yk1s6x_f.js +0 -1
- package/.next/standalone/.next/static/chunks/3tzq0260n17ae.js +0 -6
- package/.next/standalone/.next/static/chunks/3wq9x0c-7ld5d.css +0 -2
- /package/.next/standalone/.next/static/{ZklXxyA5ygsWSQz2u_zii → nwbyWT06Ud0XdXOrVJyvS}/_buildManifest.js +0 -0
- /package/.next/standalone/.next/static/{ZklXxyA5ygsWSQz2u_zii → nwbyWT06Ud0XdXOrVJyvS}/_clientMiddlewareManifest.js +0 -0
- /package/.next/standalone/.next/static/{ZklXxyA5ygsWSQz2u_zii → nwbyWT06Ud0XdXOrVJyvS}/_ssgManifest.js +0 -0
|
@@ -0,0 +1,2338 @@
|
|
|
1
|
+
"""The LangChain / LangGraph adapter, against the real framework.
|
|
2
|
+
|
|
3
|
+
Everything here runs a **real** graph on a **fake** chat model
|
|
4
|
+
(`GenericFakeChatModel` — no network, no key, deterministic) and then reads the
|
|
5
|
+
JSONL the writer actually produced. Nothing asserts on mock call arguments: the
|
|
6
|
+
whole failure mode this adapter exists inside is "the events looked right in a
|
|
7
|
+
mock and were wrong on disk".
|
|
8
|
+
|
|
9
|
+
Two of these tests are worth more than the rest put together:
|
|
10
|
+
|
|
11
|
+
* `test_every_override_still_exists_on_its_framework_base` — if upstream renames
|
|
12
|
+
a callback, our override becomes **dead code that is never called** and every
|
|
13
|
+
behavioural test below still passes, because they only ever assert on events
|
|
14
|
+
we did emit. Reflection over the class is the only thing that catches it.
|
|
15
|
+
* `test_the_node_filter_still_matches_real_langgraph_metadata` — the same
|
|
16
|
+
problem one layer down. The node/inner-runnable filter is a string comparison
|
|
17
|
+
against `metadata["langgraph_node"]`; if that key moves, every node silently
|
|
18
|
+
stops being a hook and the timeline just gets shorter.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
import inspect
|
|
22
|
+
import json
|
|
23
|
+
import logging
|
|
24
|
+
import operator
|
|
25
|
+
import os
|
|
26
|
+
from typing import Annotated, TypedDict
|
|
27
|
+
|
|
28
|
+
import pytest
|
|
29
|
+
|
|
30
|
+
import failproofai_sdk
|
|
31
|
+
from failproofai_sdk.integrations import _compat, _core
|
|
32
|
+
|
|
33
|
+
pytestmark = pytest.mark.framework
|
|
34
|
+
|
|
35
|
+
# `importorskip` is fail-open: misspell the module and every test here skips
|
|
36
|
+
# while CI stays green having tested nothing. The framework CI leg sets this
|
|
37
|
+
# env var, which turns the skip into an import error.
|
|
38
|
+
if os.environ.get("AGENTEYE_TESTS_REQUIRE_FRAMEWORKS"):
|
|
39
|
+
import langchain_core # noqa: F401
|
|
40
|
+
import langgraph # noqa: F401
|
|
41
|
+
else:
|
|
42
|
+
pytest.importorskip("langchain_core")
|
|
43
|
+
pytest.importorskip("langgraph")
|
|
44
|
+
|
|
45
|
+
from langchain_core.documents import Document # noqa: E402
|
|
46
|
+
from langchain_core.language_models.fake_chat_models import ( # noqa: E402
|
|
47
|
+
FakeListChatModel,
|
|
48
|
+
GenericFakeChatModel,
|
|
49
|
+
)
|
|
50
|
+
from langchain_core.messages import AIMessage, HumanMessage, ToolMessage # noqa: E402
|
|
51
|
+
from langchain_core.retrievers import BaseRetriever # noqa: E402
|
|
52
|
+
from langchain_core.runnables import RunnableConfig, RunnableLambda # noqa: E402
|
|
53
|
+
from langchain_core.tools import tool # noqa: E402
|
|
54
|
+
from langgraph.checkpoint.memory import InMemorySaver # noqa: E402
|
|
55
|
+
from langgraph.graph import END, START, StateGraph # noqa: E402
|
|
56
|
+
from langgraph.prebuilt import ToolNode # noqa: E402
|
|
57
|
+
from langgraph.types import Command, Send, interrupt # noqa: E402
|
|
58
|
+
|
|
59
|
+
from failproofai_sdk.integrations import langchain as adapter # noqa: E402
|
|
60
|
+
|
|
61
|
+
WATCHED_LOGGERS = ("failproofai_sdk.integrations", "langchain_core.callbacks.manager")
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
# ---------------------------------------------------------------------------
|
|
65
|
+
# Fixtures
|
|
66
|
+
# ---------------------------------------------------------------------------
|
|
67
|
+
|
|
68
|
+
class _Sink(logging.Handler):
|
|
69
|
+
"""Collects anything the adapter (or LangChain) swallowed.
|
|
70
|
+
|
|
71
|
+
LangChain firewalls handler exceptions in `handle_event` — it catches, logs
|
|
72
|
+
a WARNING and carries on — and `_core.safe` does the same one layer in. So a
|
|
73
|
+
broken translator produces a **green test** and a log line. This turns that
|
|
74
|
+
log line into a failure, which is the only way these tests mean anything.
|
|
75
|
+
"""
|
|
76
|
+
|
|
77
|
+
def __init__(self):
|
|
78
|
+
super().__init__(level=logging.WARNING)
|
|
79
|
+
self.records = []
|
|
80
|
+
self.allow = False
|
|
81
|
+
|
|
82
|
+
def emit(self, record):
|
|
83
|
+
self.records.append(record)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@pytest.fixture(autouse=True)
|
|
87
|
+
def sink():
|
|
88
|
+
handler = _Sink()
|
|
89
|
+
loggers = [logging.getLogger(name) for name in WATCHED_LOGGERS]
|
|
90
|
+
for logger in loggers:
|
|
91
|
+
logger.addHandler(handler)
|
|
92
|
+
try:
|
|
93
|
+
yield handler
|
|
94
|
+
finally:
|
|
95
|
+
for logger in loggers:
|
|
96
|
+
logger.removeHandler(handler)
|
|
97
|
+
if not handler.allow and handler.records:
|
|
98
|
+
pytest.fail(
|
|
99
|
+
"instrumentation failure was swallowed and only logged:\n"
|
|
100
|
+
+ "\n".join(handler.format(r) for r in handler.records)
|
|
101
|
+
)
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
@pytest.fixture(autouse=True)
|
|
105
|
+
def _adapter_state(monkeypatch, tmp_path):
|
|
106
|
+
import shutil
|
|
107
|
+
|
|
108
|
+
from failproofai_sdk import _runtime
|
|
109
|
+
|
|
110
|
+
monkeypatch.delenv("FAILPROOFAI_SDK_STRICT", raising=False)
|
|
111
|
+
monkeypatch.delenv("FAILPROOFAI_SDK_STRICT_INTEGRATIONS", raising=False)
|
|
112
|
+
monkeypatch.delenv(adapter.ENV_VAR, raising=False)
|
|
113
|
+
_core.set_strict(None)
|
|
114
|
+
_compat.set_strict_integrations(None)
|
|
115
|
+
_core.reset_failures()
|
|
116
|
+
_compat.reset_warnings()
|
|
117
|
+
# Instrumentation is process-global and `instrument()` with no argument
|
|
118
|
+
# installs every *detected* framework — so another test file that
|
|
119
|
+
# auto-detected can leave this adapter active, and `instrument("langchain")`
|
|
120
|
+
# would then correctly return () and this file would test nothing.
|
|
121
|
+
failproofai_sdk.uninstrument()
|
|
122
|
+
# The writer's own thread must not flush while a test is running: filenames
|
|
123
|
+
# have millisecond resolution, so two flushes in the same millisecond
|
|
124
|
+
# clobber each other and the test reads half its events.
|
|
125
|
+
_runtime.writer.set_flush_interval(3600)
|
|
126
|
+
# A previous test can legitimately emit *after* it read its events —
|
|
127
|
+
# `uninstrument()` closes spans that were still open, which is the point.
|
|
128
|
+
# Drain and discard those before this test starts, or they land in this
|
|
129
|
+
# test's directory and every assertion about "the first agent_end" is wrong.
|
|
130
|
+
_runtime.writer.flush_now()
|
|
131
|
+
shutil.rmtree(tmp_path / "events", ignore_errors=True)
|
|
132
|
+
yield
|
|
133
|
+
failproofai_sdk.uninstrument()
|
|
134
|
+
_runtime.writer.flush_now()
|
|
135
|
+
_core.reset_failures()
|
|
136
|
+
_compat.reset_warnings()
|
|
137
|
+
_core.set_strict(None)
|
|
138
|
+
_compat.set_strict_integrations(None)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
@pytest.fixture()
|
|
142
|
+
def instrumented():
|
|
143
|
+
assert failproofai_sdk.instrument("langchain") == ("langchain",)
|
|
144
|
+
yield
|
|
145
|
+
failproofai_sdk.uninstrument("langchain")
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def read_events(tmp_path):
|
|
149
|
+
"""Flush and read what actually reached disk, in emission order."""
|
|
150
|
+
failproofai_sdk._writer.flush_now()
|
|
151
|
+
rows = []
|
|
152
|
+
for path in sorted((tmp_path / "events").glob("*.jsonl")):
|
|
153
|
+
for line in path.read_text(encoding="utf-8").splitlines():
|
|
154
|
+
if line.strip():
|
|
155
|
+
rows.append(json.loads(line))
|
|
156
|
+
return rows
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def types_of(rows):
|
|
160
|
+
return [row["type"] for row in rows]
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
def only(rows, *kinds):
|
|
164
|
+
return [row for row in rows if row["type"] in kinds]
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
# ---------------------------------------------------------------------------
|
|
168
|
+
# Graphs
|
|
169
|
+
# ---------------------------------------------------------------------------
|
|
170
|
+
|
|
171
|
+
@tool
|
|
172
|
+
def adder(a: int, b: int) -> int:
|
|
173
|
+
"""Add two numbers."""
|
|
174
|
+
return a + b
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
@tool
|
|
178
|
+
def exploder(x: int) -> int:
|
|
179
|
+
"""Always fails."""
|
|
180
|
+
raise RuntimeError("tool boom")
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
class State(TypedDict):
|
|
184
|
+
messages: Annotated[list, operator.add]
|
|
185
|
+
vals: Annotated[list, operator.add]
|
|
186
|
+
answer: str
|
|
187
|
+
n: int
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def tool_calling_message(**kwargs):
|
|
191
|
+
return AIMessage(
|
|
192
|
+
"planned",
|
|
193
|
+
tool_calls=[{"name": "adder", "args": {"a": 1, "b": 2}, "id": "call_abc"}],
|
|
194
|
+
usage_metadata={"input_tokens": 11, "output_tokens": 5, "total_tokens": 16},
|
|
195
|
+
**kwargs,
|
|
196
|
+
)
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def fake_model(*messages):
|
|
200
|
+
return GenericFakeChatModel(messages=iter(list(messages)))
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def subgraph():
|
|
204
|
+
sg = StateGraph(State)
|
|
205
|
+
sg.add_node("sub_step", lambda state: {"vals": ["sub"]})
|
|
206
|
+
sg.add_edge(START, "sub_step")
|
|
207
|
+
sg.add_edge("sub_step", END)
|
|
208
|
+
return sg.compile(name="child_graph")
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def build_graph(*, model=None, checkpointer=None, name="root_graph"):
|
|
212
|
+
"""plan -> tools -> child(subgraph) -> fan-out -> ask(interrupt) -> END."""
|
|
213
|
+
model = model or fake_model(tool_calling_message(), AIMessage("again"))
|
|
214
|
+
|
|
215
|
+
def plan(state):
|
|
216
|
+
reply = model.invoke(state["messages"])
|
|
217
|
+
return {"messages": [reply], "n": state["n"] + 1, "vals": ["plan"]}
|
|
218
|
+
|
|
219
|
+
def fan(state):
|
|
220
|
+
return [Send("worker", {"i": i}) for i in range(2)]
|
|
221
|
+
|
|
222
|
+
def ask(state):
|
|
223
|
+
return {"answer": str(interrupt({"prompt": "approve?", "options": ["y", "n"]}))}
|
|
224
|
+
|
|
225
|
+
graph = StateGraph(State)
|
|
226
|
+
graph.add_node("plan", plan)
|
|
227
|
+
graph.add_node("tools", ToolNode([adder]))
|
|
228
|
+
graph.add_node("child", subgraph())
|
|
229
|
+
graph.add_node("worker", lambda state: {"vals": ["w"]})
|
|
230
|
+
graph.add_node("ask", ask)
|
|
231
|
+
graph.add_edge(START, "plan")
|
|
232
|
+
graph.add_edge("plan", "tools")
|
|
233
|
+
graph.add_edge("tools", "child")
|
|
234
|
+
graph.add_conditional_edges("child", fan, ["worker"])
|
|
235
|
+
graph.add_edge("worker", "ask")
|
|
236
|
+
graph.add_edge("ask", END)
|
|
237
|
+
return graph.compile(name=name, checkpointer=checkpointer)
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def build_simple(nodes, *, name="simple", checkpointer=None, edges=None):
|
|
241
|
+
graph = StateGraph(State)
|
|
242
|
+
previous = START
|
|
243
|
+
for node_name, fn in nodes:
|
|
244
|
+
graph.add_node(node_name, fn)
|
|
245
|
+
graph.add_edge(previous, node_name)
|
|
246
|
+
previous = node_name
|
|
247
|
+
graph.add_edge(previous, END)
|
|
248
|
+
return graph.compile(name=name, checkpointer=checkpointer)
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def empty_state(**kwargs):
|
|
252
|
+
base = {"messages": [HumanMessage("hi")], "vals": [], "answer": "", "n": 0}
|
|
253
|
+
base.update(kwargs)
|
|
254
|
+
return base
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def build_looping(model):
|
|
258
|
+
"""plan -> check -> (plan | END). Exercises a node visited more than once."""
|
|
259
|
+
|
|
260
|
+
def plan(state):
|
|
261
|
+
model.invoke(state["messages"])
|
|
262
|
+
return {"n": state["n"] + 1, "vals": ["plan"]}
|
|
263
|
+
|
|
264
|
+
def check(state):
|
|
265
|
+
return {"vals": ["check"]}
|
|
266
|
+
|
|
267
|
+
graph = StateGraph(State)
|
|
268
|
+
graph.add_node("plan", plan)
|
|
269
|
+
graph.add_node("check", check)
|
|
270
|
+
graph.add_edge(START, "plan")
|
|
271
|
+
graph.add_edge("plan", "check")
|
|
272
|
+
graph.add_conditional_edges(
|
|
273
|
+
"check", lambda s: "plan" if s["n"] < 3 else END, {"plan": "plan", END: END}
|
|
274
|
+
)
|
|
275
|
+
return graph.compile(name="looper")
|
|
276
|
+
|
|
277
|
+
|
|
278
|
+
# ---------------------------------------------------------------------------
|
|
279
|
+
# Shape of a representative run
|
|
280
|
+
# ---------------------------------------------------------------------------
|
|
281
|
+
|
|
282
|
+
def test_event_type_sequence_for_a_representative_run(tmp_path, instrumented):
|
|
283
|
+
model = fake_model(tool_calling_message())
|
|
284
|
+
|
|
285
|
+
def plan(state):
|
|
286
|
+
return {"messages": [model.invoke(state["messages"])], "vals": ["plan"]}
|
|
287
|
+
|
|
288
|
+
def act(state):
|
|
289
|
+
adder.invoke({"a": 1, "b": 2})
|
|
290
|
+
return {"vals": ["act"]}
|
|
291
|
+
|
|
292
|
+
app = build_simple([("plan", plan), ("act", act)], name="pipeline")
|
|
293
|
+
app.invoke(empty_state(), config={"configurable": {"thread_id": "seq"}})
|
|
294
|
+
|
|
295
|
+
assert types_of(read_events(tmp_path)) == [
|
|
296
|
+
"agent_start",
|
|
297
|
+
"hook_triggered", # plan
|
|
298
|
+
"model_request",
|
|
299
|
+
"model_response",
|
|
300
|
+
"hook_completed",
|
|
301
|
+
"hook_triggered", # act
|
|
302
|
+
"tool_use",
|
|
303
|
+
"tool_result",
|
|
304
|
+
"hook_completed",
|
|
305
|
+
"agent_end",
|
|
306
|
+
]
|
|
307
|
+
|
|
308
|
+
|
|
309
|
+
def test_the_root_agent_start_is_the_sessions_first_event(tmp_path, instrumented):
|
|
310
|
+
build_graph(checkpointer=InMemorySaver()).invoke(
|
|
311
|
+
empty_state(), config={"configurable": {"thread_id": "first"}}
|
|
312
|
+
)
|
|
313
|
+
rows = read_events(tmp_path)
|
|
314
|
+
assert rows[0]["type"] == "agent_start"
|
|
315
|
+
assert rows[0]["agent_id"] == "root_graph"
|
|
316
|
+
assert rows[0].get("parent_id") is None
|
|
317
|
+
# ...and it is the only root agent_start in the session.
|
|
318
|
+
roots = [r for r in rows if r["type"] == "agent_start" and r.get("parent_id") is None]
|
|
319
|
+
assert len(roots) == 1
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def test_every_event_carries_one_session_id_and_the_framework(tmp_path, instrumented):
|
|
323
|
+
build_graph(checkpointer=InMemorySaver()).invoke(
|
|
324
|
+
empty_state(), config={"configurable": {"thread_id": "sess"}}
|
|
325
|
+
)
|
|
326
|
+
rows = read_events(tmp_path)
|
|
327
|
+
assert rows
|
|
328
|
+
assert {r["session_id"] for r in rows} == {"sess"}
|
|
329
|
+
assert {r["framework"] for r in rows} == {"langchain"}
|
|
330
|
+
assert all(r["framework_version"] for r in rows)
|
|
331
|
+
assert all(r["integration_version"] for r in rows)
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def test_agent_ids_are_human_readable_names_never_uuids(tmp_path, instrumented):
|
|
335
|
+
build_graph(checkpointer=InMemorySaver()).invoke(
|
|
336
|
+
empty_state(), config={"configurable": {"thread_id": "names"}}
|
|
337
|
+
)
|
|
338
|
+
rows = read_events(tmp_path)
|
|
339
|
+
agent_ids = {r["agent_id"] for r in rows}
|
|
340
|
+
assert agent_ids == {"root_graph", "root_graph/child"}
|
|
341
|
+
for value in agent_ids:
|
|
342
|
+
assert not _looks_like_a_uuid(value), value
|
|
343
|
+
# Hook names are node names too — `hook_name` is its own facet on /hooks and
|
|
344
|
+
# a uuid there is just as poisonous as one in agent_id.
|
|
345
|
+
for row in only(rows, "hook_triggered", "hook_completed"):
|
|
346
|
+
assert not _looks_like_a_uuid(row["hook_name"])
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def _looks_like_a_uuid(value):
|
|
350
|
+
import uuid
|
|
351
|
+
|
|
352
|
+
try:
|
|
353
|
+
uuid.UUID(str(value))
|
|
354
|
+
except (ValueError, AttributeError, TypeError):
|
|
355
|
+
return False
|
|
356
|
+
return True
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
def test_no_event_carries_an_extra_that_would_shadow_a_declared_field(tmp_path, instrumented):
|
|
360
|
+
build_graph(checkpointer=InMemorySaver()).invoke(
|
|
361
|
+
empty_state(), config={"configurable": {"thread_id": "shadow"}}
|
|
362
|
+
)
|
|
363
|
+
rows = read_events(tmp_path)
|
|
364
|
+
assert rows
|
|
365
|
+
# `_schema._build()` ends with `result.update(extra)`, so an extra called
|
|
366
|
+
# `tool_name`, `model`, `outcome` or `input_tokens` silently OVERWRITES the
|
|
367
|
+
# declared field — and therefore the promoted column and the
|
|
368
|
+
# server's computed summary — while every behavioural test still passes.
|
|
369
|
+
# The allowed key set is derived from `_schema`'s own dataclasses, so adding
|
|
370
|
+
# a field there cannot leave a stale copy here.
|
|
371
|
+
declared = _declared_fields_by_type()
|
|
372
|
+
assert set(declared) == _SCHEMA_TYPES
|
|
373
|
+
for row in rows:
|
|
374
|
+
allowed = declared[row["type"]] | _core.ALLOWED_TOP_LEVEL
|
|
375
|
+
for key in row:
|
|
376
|
+
if key.startswith("fw_"):
|
|
377
|
+
assert key not in _core.FORBIDDEN_EXTRAS, (row["type"], key)
|
|
378
|
+
continue
|
|
379
|
+
assert key in allowed, (
|
|
380
|
+
"%s carries %r, which is not a field of that event — if it is an "
|
|
381
|
+
"extra it must be namespaced fw_*, or it silently shadows a "
|
|
382
|
+
"declared field" % (row["type"], key)
|
|
383
|
+
)
|
|
384
|
+
|
|
385
|
+
|
|
386
|
+
def _declared_fields_by_type():
|
|
387
|
+
import dataclasses
|
|
388
|
+
import re
|
|
389
|
+
|
|
390
|
+
from failproofai_sdk import _schema
|
|
391
|
+
|
|
392
|
+
out = {}
|
|
393
|
+
for name, obj in vars(_schema).items():
|
|
394
|
+
if not (dataclasses.is_dataclass(obj) and isinstance(obj, type)):
|
|
395
|
+
continue
|
|
396
|
+
event_type = re.sub(r"(?<!^)(?=[A-Z])", "_", name[: -len("Event")]).lower()
|
|
397
|
+
fields = {f.name for f in dataclasses.fields(obj)} - {"extra_fields"}
|
|
398
|
+
out[event_type] = fields | {"type", "environment"}
|
|
399
|
+
return out
|
|
400
|
+
|
|
401
|
+
|
|
402
|
+
_SCHEMA_TYPES = {
|
|
403
|
+
"agent_start", "agent_end", "agent_pause", "agent_resume",
|
|
404
|
+
"tool_use", "tool_result", "model_request", "model_response",
|
|
405
|
+
"hook_triggered", "hook_completed", "error",
|
|
406
|
+
"human_wait", "human_input", "human_pause", "human_interrupt",
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
|
|
410
|
+
def test_only_the_fifteen_known_event_types_are_emitted(tmp_path, instrumented):
|
|
411
|
+
app = build_graph(checkpointer=InMemorySaver())
|
|
412
|
+
config = {"configurable": {"thread_id": "vocab"}}
|
|
413
|
+
app.invoke(empty_state(), config=config)
|
|
414
|
+
app.invoke(Command(resume="yes"), config=config)
|
|
415
|
+
assert set(types_of(read_events(tmp_path))) <= _SCHEMA_TYPES
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
# ---------------------------------------------------------------------------
|
|
419
|
+
# Models
|
|
420
|
+
# ---------------------------------------------------------------------------
|
|
421
|
+
|
|
422
|
+
def test_model_events_pair_on_request_id_and_always_carry_an_int_duration(
|
|
423
|
+
tmp_path, instrumented
|
|
424
|
+
):
|
|
425
|
+
app = build_looping(fake_model(*[AIMessage("t%d" % i) for i in range(5)]))
|
|
426
|
+
app.invoke(empty_state(), config={"configurable": {"thread_id": "models"}})
|
|
427
|
+
rows = read_events(tmp_path)
|
|
428
|
+
|
|
429
|
+
requests = only(rows, "model_request")
|
|
430
|
+
responses = only(rows, "model_response")
|
|
431
|
+
assert len(requests) == len(responses) == 3 # the loop runs `plan` three times
|
|
432
|
+
assert [r["request_id"] for r in requests] == [r["request_id"] for r in responses]
|
|
433
|
+
assert len({r["request_id"] for r in requests}) == 3
|
|
434
|
+
|
|
435
|
+
for response in responses:
|
|
436
|
+
# Not guarded by the SDK, and `durationOf` prefers the closing event's
|
|
437
|
+
# value — which is what keeps model durations correct even though the
|
|
438
|
+
# execution graph pairs model events FIFO per agent_id. Must be an int:
|
|
439
|
+
# the server's JSON parser drops floats and NULLs the u32 column.
|
|
440
|
+
assert isinstance(response["duration_ms"], int)
|
|
441
|
+
assert not isinstance(response["duration_ms"], bool)
|
|
442
|
+
assert response["duration_ms"] >= 0
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
def test_model_name_and_tokens_are_normalized(tmp_path, instrumented):
|
|
446
|
+
app = build_simple(
|
|
447
|
+
[("plan", lambda s: {"messages": [_MODEL.invoke(s["messages"])], "vals": ["p"]})],
|
|
448
|
+
name="tokens",
|
|
449
|
+
)
|
|
450
|
+
app.invoke(empty_state(), config={"configurable": {"thread_id": "tok"}})
|
|
451
|
+
response = only(read_events(tmp_path), "model_response")[0]
|
|
452
|
+
assert response["model"] == "GenericFakeChatModel"
|
|
453
|
+
assert response["input_tokens"] == 11
|
|
454
|
+
assert response["output_tokens"] == 5
|
|
455
|
+
# Shipped as a dict as well: both the server summary and the dashboard fall
|
|
456
|
+
# back to payload.usage when the promoted columns are absent.
|
|
457
|
+
assert response["usage"] == {"input_tokens": 11, "output_tokens": 5, "total_tokens": 16}
|
|
458
|
+
|
|
459
|
+
|
|
460
|
+
_MODEL = fake_model(tool_calling_message(), AIMessage("x"), AIMessage("y"))
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def test_model_request_carries_normalized_messages_not_flattened_prompts(
|
|
464
|
+
tmp_path, instrumented
|
|
465
|
+
):
|
|
466
|
+
model = fake_model(AIMessage("ok"))
|
|
467
|
+
app = build_simple(
|
|
468
|
+
[("plan", lambda s: {"messages": [model.invoke(s["messages"])], "vals": ["p"]})],
|
|
469
|
+
name="msgs",
|
|
470
|
+
)
|
|
471
|
+
app.invoke(empty_state(), config={"configurable": {"thread_id": "msg"}})
|
|
472
|
+
request = only(read_events(tmp_path), "model_request")[0]
|
|
473
|
+
# `_create_chat_model_run` flattens messages to "Human: hi" before they
|
|
474
|
+
# reach the Run object, which loses the roles. We capture them from
|
|
475
|
+
# `on_chat_model_start`, where they are still BaseMessage objects.
|
|
476
|
+
assert request["messages"] == [{"role": "user", "content": "hi"}]
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
def test_streaming_never_emits_per_token_events(tmp_path, instrumented):
|
|
480
|
+
model = FakeListChatModel(responses=["hello there"])
|
|
481
|
+
app = build_simple(
|
|
482
|
+
[("stream", lambda s: {"vals": [c.content for c in model.stream(s["messages"])]})],
|
|
483
|
+
name="streamer",
|
|
484
|
+
)
|
|
485
|
+
app.invoke(empty_state(), config={"configurable": {"thread_id": "stream"}})
|
|
486
|
+
rows = read_events(tmp_path)
|
|
487
|
+
# A 500-token response must not become 500 rows against a five-lane rail.
|
|
488
|
+
assert len(only(rows, "model_request")) == 1
|
|
489
|
+
assert len(only(rows, "model_response")) == 1
|
|
490
|
+
response = only(rows, "model_response")[0]
|
|
491
|
+
assert response["fw_streamed"] is True
|
|
492
|
+
assert response["fw_chunks"] >= len("hello there")
|
|
493
|
+
assert isinstance(response["fw_ttft_ms"], int)
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
def test_a_failed_model_call_is_reported_on_the_model_span(tmp_path, instrumented, sink):
|
|
497
|
+
class Boom(GenericFakeChatModel):
|
|
498
|
+
def _generate(self, *args, **kwargs):
|
|
499
|
+
raise RuntimeError("model 429")
|
|
500
|
+
|
|
501
|
+
model = Boom(messages=iter([AIMessage("never")]))
|
|
502
|
+
|
|
503
|
+
def plan(state):
|
|
504
|
+
try:
|
|
505
|
+
model.invoke(state["messages"])
|
|
506
|
+
except RuntimeError:
|
|
507
|
+
return {"vals": ["caught"]}
|
|
508
|
+
return {"vals": ["nope"]}
|
|
509
|
+
|
|
510
|
+
build_simple([("plan", plan)], name="modelfail").invoke(
|
|
511
|
+
empty_state(), config={"configurable": {"thread_id": "mf"}}
|
|
512
|
+
)
|
|
513
|
+
rows = read_events(tmp_path)
|
|
514
|
+
response = only(rows, "model_response")[0]
|
|
515
|
+
assert response["stop_reason"] == "error"
|
|
516
|
+
assert "model 429" in response["error"]
|
|
517
|
+
assert isinstance(response["duration_ms"], int)
|
|
518
|
+
# The node caught it, so nothing above owns the failure and no standalone
|
|
519
|
+
# `error` event may appear — `sessionSummary.errorCount` would double-count.
|
|
520
|
+
assert not only(rows, "error")
|
|
521
|
+
assert only(rows, "agent_end")[0]["outcome"] == "success"
|
|
522
|
+
|
|
523
|
+
|
|
524
|
+
# ---------------------------------------------------------------------------
|
|
525
|
+
# Tools and retrievers
|
|
526
|
+
# ---------------------------------------------------------------------------
|
|
527
|
+
|
|
528
|
+
def test_tool_events_pair_on_tool_call_id_and_carry_a_duration(tmp_path, instrumented):
|
|
529
|
+
model = fake_model(tool_calling_message())
|
|
530
|
+
|
|
531
|
+
def plan(state):
|
|
532
|
+
return {"messages": [model.invoke(state["messages"])], "vals": ["p"]}
|
|
533
|
+
|
|
534
|
+
graph = StateGraph(State)
|
|
535
|
+
graph.add_node("plan", plan)
|
|
536
|
+
graph.add_node("tools", ToolNode([adder]))
|
|
537
|
+
graph.add_edge(START, "plan")
|
|
538
|
+
graph.add_edge("plan", "tools")
|
|
539
|
+
graph.add_edge("tools", END)
|
|
540
|
+
graph.compile(name="tools_graph").invoke(
|
|
541
|
+
empty_state(), config={"configurable": {"thread_id": "tools"}}
|
|
542
|
+
)
|
|
543
|
+
|
|
544
|
+
rows = read_events(tmp_path)
|
|
545
|
+
use = only(rows, "tool_use")[0]
|
|
546
|
+
result = only(rows, "tool_result")[0]
|
|
547
|
+
assert use["tool_call_id"] == result["tool_call_id"]
|
|
548
|
+
# The **LLM-issued** id, not our run id: this is what makes our events line
|
|
549
|
+
# up with the provider's logs and with the assistant message's tool_calls.
|
|
550
|
+
assert use["tool_call_id"] == "call_abc"
|
|
551
|
+
assert use["tool_name"] == result["tool_name"] == "adder"
|
|
552
|
+
assert isinstance(result["duration_ms"], int)
|
|
553
|
+
assert result.get("error") is None
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
def test_a_failed_tool_is_reported_on_the_tool_span_only(tmp_path, instrumented):
|
|
557
|
+
def act(state):
|
|
558
|
+
try:
|
|
559
|
+
exploder.invoke({"x": 1})
|
|
560
|
+
except RuntimeError:
|
|
561
|
+
return {"vals": ["caught"]}
|
|
562
|
+
return {"vals": ["nope"]}
|
|
563
|
+
|
|
564
|
+
build_simple([("act", act)], name="toolfail").invoke(
|
|
565
|
+
empty_state(), config={"configurable": {"thread_id": "tf"}}
|
|
566
|
+
)
|
|
567
|
+
rows = read_events(tmp_path)
|
|
568
|
+
result = only(rows, "tool_result")[0]
|
|
569
|
+
assert "tool boom" in result["error"]
|
|
570
|
+
# One error, on the span that owns it. No standalone `error` event, and the
|
|
571
|
+
# run itself succeeded because the node handled the failure.
|
|
572
|
+
assert not only(rows, "error")
|
|
573
|
+
assert only(rows, "agent_end")[0]["outcome"] == "success"
|
|
574
|
+
assert only(rows, "hook_completed")[0]["outcome"] == "success"
|
|
575
|
+
|
|
576
|
+
|
|
577
|
+
def test_retriever_output_is_summarized_never_the_document_text(tmp_path, instrumented):
|
|
578
|
+
class Retriever(BaseRetriever):
|
|
579
|
+
def _get_relevant_documents(self, query, *, run_manager=None):
|
|
580
|
+
return [
|
|
581
|
+
Document(page_content="SECRET" * 500, metadata={"source": "a.txt"}),
|
|
582
|
+
Document(page_content="SECRET" * 500, metadata={"source": "b.txt"}),
|
|
583
|
+
]
|
|
584
|
+
|
|
585
|
+
retriever = Retriever()
|
|
586
|
+
build_simple(
|
|
587
|
+
[("fetch", lambda s: {"vals": [len(retriever.invoke("q"))]})], name="rag"
|
|
588
|
+
).invoke(empty_state(), config={"configurable": {"thread_id": "rag"}})
|
|
589
|
+
|
|
590
|
+
rows = read_events(tmp_path)
|
|
591
|
+
use = only(rows, "tool_use")[0]
|
|
592
|
+
result = only(rows, "tool_result")[0]
|
|
593
|
+
assert use["tool_name"] == "retriever:Retriever"
|
|
594
|
+
assert result["output"] == {"n": 2, "sources": ["a.txt", "b.txt"]}
|
|
595
|
+
assert "SECRET" not in json.dumps(rows)
|
|
596
|
+
|
|
597
|
+
|
|
598
|
+
# ---------------------------------------------------------------------------
|
|
599
|
+
# Graph structure
|
|
600
|
+
# ---------------------------------------------------------------------------
|
|
601
|
+
|
|
602
|
+
def test_a_langgraph_node_is_a_hook_not_a_nested_agent(tmp_path, instrumented):
|
|
603
|
+
build_graph(checkpointer=InMemorySaver()).invoke(
|
|
604
|
+
empty_state(), config={"configurable": {"thread_id": "hooks"}}
|
|
605
|
+
)
|
|
606
|
+
rows = read_events(tmp_path)
|
|
607
|
+
hooks = {r["hook_name"] for r in only(rows, "hook_triggered")}
|
|
608
|
+
# `sub_step` is the subgraph's own node — also a hook, under the nested agent.
|
|
609
|
+
assert hooks == {"plan", "tools", "child", "worker", "ask", "sub_step"}
|
|
610
|
+
for row in only(rows, "hook_triggered"):
|
|
611
|
+
assert row["trigger_event"] == "graph_node"
|
|
612
|
+
# Nodes must never inflate agent_id: it is a LowCardinality column and the
|
|
613
|
+
# primary facet, and `agent_sessions.agent_id = any(...)` would label the
|
|
614
|
+
# session with whichever node happened to run first.
|
|
615
|
+
assert "plan" not in {r["agent_id"] for r in rows}
|
|
616
|
+
|
|
617
|
+
|
|
618
|
+
def test_a_compiled_subgraph_becomes_a_nested_agent(tmp_path, instrumented):
|
|
619
|
+
build_graph(checkpointer=InMemorySaver()).invoke(
|
|
620
|
+
empty_state(), config={"configurable": {"thread_id": "sub"}}
|
|
621
|
+
)
|
|
622
|
+
rows = read_events(tmp_path)
|
|
623
|
+
nested = [r for r in only(rows, "agent_start") if r.get("parent_id")]
|
|
624
|
+
assert len(nested) == 1
|
|
625
|
+
assert nested[0]["agent_id"] == "root_graph/child"
|
|
626
|
+
assert nested[0]["parent_id"] == "root_graph"
|
|
627
|
+
# The subgraph's own node runs under the nested agent, and the agent closes.
|
|
628
|
+
inner = [r for r in rows if r["agent_id"] == "root_graph/child"]
|
|
629
|
+
assert types_of(inner) == ["agent_start", "hook_triggered", "hook_completed", "agent_end"]
|
|
630
|
+
assert inner[1]["hook_name"] == "sub_step"
|
|
631
|
+
|
|
632
|
+
|
|
633
|
+
def test_intermediate_runnables_and_edge_functions_emit_nothing(tmp_path, instrumented):
|
|
634
|
+
rows = []
|
|
635
|
+
|
|
636
|
+
def plan(state):
|
|
637
|
+
return {"vals": ["p"]}
|
|
638
|
+
|
|
639
|
+
graph = StateGraph(State)
|
|
640
|
+
graph.add_node("plan", plan)
|
|
641
|
+
graph.add_node("done", lambda s: {"vals": ["d"]})
|
|
642
|
+
graph.add_edge(START, "plan")
|
|
643
|
+
graph.add_conditional_edges(
|
|
644
|
+
"plan", _named_edge, {"done": "done", END: END}
|
|
645
|
+
)
|
|
646
|
+
graph.add_edge("done", END)
|
|
647
|
+
graph.compile(name="edges").invoke(
|
|
648
|
+
empty_state(), config={"configurable": {"thread_id": "edges"}}
|
|
649
|
+
)
|
|
650
|
+
rows = read_events(tmp_path)
|
|
651
|
+
names = {r["hook_name"] for r in only(rows, "hook_triggered")}
|
|
652
|
+
# `_named_edge` is a Runnable with its own run and it inherits the node's
|
|
653
|
+
# `langgraph_node` metadata; only `run.name == metadata["langgraph_node"]`
|
|
654
|
+
# keeps it out. Emitting it would bury the timeline in machinery.
|
|
655
|
+
assert names == {"plan", "done"}
|
|
656
|
+
|
|
657
|
+
|
|
658
|
+
def _named_edge(state):
|
|
659
|
+
return "done"
|
|
660
|
+
|
|
661
|
+
|
|
662
|
+
def test_a_node_visited_repeatedly_produces_one_hook_pair_per_visit(tmp_path, instrumented):
|
|
663
|
+
app = build_looping(fake_model(*[AIMessage("t%d" % i) for i in range(5)]))
|
|
664
|
+
app.invoke(empty_state(), config={"configurable": {"thread_id": "loop"}})
|
|
665
|
+
rows = read_events(tmp_path)
|
|
666
|
+
plans = [r for r in only(rows, "hook_triggered") if r["hook_name"] == "plan"]
|
|
667
|
+
assert len(plans) == 3
|
|
668
|
+
assert len({r["hook_id"] for r in plans}) == 3
|
|
669
|
+
completed = [r for r in only(rows, "hook_completed") if r["hook_name"] == "plan"]
|
|
670
|
+
assert {r["hook_id"] for r in plans} == {r["hook_id"] for r in completed}
|
|
671
|
+
|
|
672
|
+
|
|
673
|
+
def test_a_parallel_fan_out_produces_one_hook_pair_per_branch(tmp_path, instrumented):
|
|
674
|
+
build_graph(checkpointer=InMemorySaver()).invoke(
|
|
675
|
+
empty_state(), config={"configurable": {"thread_id": "fan"}}
|
|
676
|
+
)
|
|
677
|
+
rows = read_events(tmp_path)
|
|
678
|
+
workers = [r for r in only(rows, "hook_triggered") if r["hook_name"] == "worker"]
|
|
679
|
+
assert len(workers) == 2
|
|
680
|
+
assert len({r["hook_id"] for r in workers}) == 2
|
|
681
|
+
|
|
682
|
+
|
|
683
|
+
def test_every_event_belongs_to_an_agent_whose_start_is_open(tmp_path, instrumented):
|
|
684
|
+
app = build_graph(checkpointer=InMemorySaver())
|
|
685
|
+
config = {"configurable": {"thread_id": "open"}}
|
|
686
|
+
app.invoke(empty_state(), config=config)
|
|
687
|
+
app.invoke(Command(resume="yes"), config=config)
|
|
688
|
+
|
|
689
|
+
open_agents = set()
|
|
690
|
+
for row in read_events(tmp_path):
|
|
691
|
+
if row["type"] == "agent_start":
|
|
692
|
+
open_agents.add(row["agent_id"])
|
|
693
|
+
continue
|
|
694
|
+
# The dashboard parents every leaf to the open agent with the same
|
|
695
|
+
# agent_id and SYNTHESISES a never-ending root span when there is none.
|
|
696
|
+
assert row["agent_id"] in open_agents, row
|
|
697
|
+
if row["type"] == "agent_end":
|
|
698
|
+
open_agents.discard(row["agent_id"])
|
|
699
|
+
assert not open_agents
|
|
700
|
+
|
|
701
|
+
|
|
702
|
+
# ---------------------------------------------------------------------------
|
|
703
|
+
# Human in the loop
|
|
704
|
+
# ---------------------------------------------------------------------------
|
|
705
|
+
|
|
706
|
+
def test_interrupt_and_resume_emit_both_pairs_in_order(tmp_path, instrumented):
|
|
707
|
+
app = build_graph(checkpointer=InMemorySaver())
|
|
708
|
+
config = {"configurable": {"thread_id": "hitl"}}
|
|
709
|
+
first = app.invoke(empty_state(), config=config)
|
|
710
|
+
assert "__interrupt__" in first
|
|
711
|
+
result = app.invoke(Command(resume="approved"), config=config)
|
|
712
|
+
assert result["answer"] == "approved"
|
|
713
|
+
|
|
714
|
+
rows = read_events(tmp_path)
|
|
715
|
+
hitl = types_of(only(rows, "human_wait", "agent_pause", "agent_resume", "human_input"))
|
|
716
|
+
# Neither pair alone is enough: only agent_pause<->agent_resume feeds
|
|
717
|
+
# `pausedMs`, and only human_wait<->human_input carries the prompt, the
|
|
718
|
+
# answer and `pendingHuman`.
|
|
719
|
+
assert hitl == ["human_wait", "agent_pause", "agent_resume", "human_input"]
|
|
720
|
+
|
|
721
|
+
wait = only(rows, "human_wait")[0]
|
|
722
|
+
pause = only(rows, "agent_pause")[0]
|
|
723
|
+
resume = only(rows, "agent_resume")[0]
|
|
724
|
+
answer = only(rows, "human_input")[0]
|
|
725
|
+
assert wait["input_id"] == pause["pause_id"] == resume["pause_id"] == answer["input_id"]
|
|
726
|
+
assert wait["prompt"] == "approve?"
|
|
727
|
+
assert wait["options"] == ["y", "n"]
|
|
728
|
+
assert answer["response"] == "approved"
|
|
729
|
+
assert isinstance(resume["duration_ms"], int)
|
|
730
|
+
|
|
731
|
+
|
|
732
|
+
def test_an_interrupt_is_control_flow_not_an_error(tmp_path, instrumented):
|
|
733
|
+
app = build_graph(checkpointer=InMemorySaver())
|
|
734
|
+
config = {"configurable": {"thread_id": "ctrl"}}
|
|
735
|
+
app.invoke(empty_state(), config=config)
|
|
736
|
+
app.invoke(Command(resume="yes"), config=config)
|
|
737
|
+
|
|
738
|
+
rows = read_events(tmp_path)
|
|
739
|
+
# LangGraph reports the GraphInterrupt through on_chain_error with no
|
|
740
|
+
# special case, so without the GraphBubbleUp check every human approval
|
|
741
|
+
# would paint a red error and a failed agent.
|
|
742
|
+
assert not only(rows, "error")
|
|
743
|
+
assert [r["outcome"] for r in only(rows, "agent_end")] == ["success", "success"]
|
|
744
|
+
asks = [r for r in only(rows, "hook_completed") if r["hook_name"] == "ask"]
|
|
745
|
+
assert [r["outcome"] for r in asks] == ["paused", "success"]
|
|
746
|
+
assert asks[0].get("error") is None
|
|
747
|
+
|
|
748
|
+
|
|
749
|
+
def test_the_agent_stays_open_across_the_pause_so_one_run_is_one_span(
|
|
750
|
+
tmp_path, instrumented
|
|
751
|
+
):
|
|
752
|
+
app = build_graph(checkpointer=InMemorySaver())
|
|
753
|
+
config = {"configurable": {"thread_id": "span"}}
|
|
754
|
+
app.invoke(empty_state(), config=config)
|
|
755
|
+
app.invoke(Command(resume="yes"), config=config)
|
|
756
|
+
|
|
757
|
+
rows = read_events(tmp_path)
|
|
758
|
+
roots = [r for r in only(rows, "agent_start") if not r.get("parent_id")]
|
|
759
|
+
# One agent_start, one agent_end, across two `.invoke()` calls. Closing the
|
|
760
|
+
# agent at the first invoke would force-close the open pause and zero out
|
|
761
|
+
# the only interval that measures how long the human took.
|
|
762
|
+
assert len(roots) == 1
|
|
763
|
+
assert len([r for r in only(rows, "agent_end") if r["agent_id"] == "root_graph"]) == 1
|
|
764
|
+
assert rows[-1]["type"] == "agent_end"
|
|
765
|
+
assert {r["session_id"] for r in rows} == {"span"}
|
|
766
|
+
|
|
767
|
+
|
|
768
|
+
def test_interrupt_events_survive_without_the_graph_lifecycle_callbacks(
|
|
769
|
+
tmp_path, monkeypatch
|
|
770
|
+
):
|
|
771
|
+
"""The exception path alone must produce the whole HITL round trip.
|
|
772
|
+
|
|
773
|
+
`GraphCallbackHandler.on_interrupt` is new in langgraph 1.2 and, as shipped,
|
|
774
|
+
is not delivered to a handler installed through `register_configure_hook` at
|
|
775
|
+
all — so the fallback is not a legacy branch, it is the load-bearing one on
|
|
776
|
+
any install where the wrap does not apply.
|
|
777
|
+
"""
|
|
778
|
+
failproofai_sdk.instrument("langchain", graph_callbacks=False)
|
|
779
|
+
try:
|
|
780
|
+
app = build_graph(checkpointer=InMemorySaver())
|
|
781
|
+
config = {"configurable": {"thread_id": "nolifecycle"}}
|
|
782
|
+
app.invoke(empty_state(), config=config)
|
|
783
|
+
app.invoke(Command(resume="yes"), config=config)
|
|
784
|
+
rows = read_events(tmp_path)
|
|
785
|
+
finally:
|
|
786
|
+
failproofai_sdk.uninstrument("langchain")
|
|
787
|
+
|
|
788
|
+
assert types_of(only(rows, "human_wait", "agent_pause", "agent_resume", "human_input")) == [
|
|
789
|
+
"human_wait",
|
|
790
|
+
"agent_pause",
|
|
791
|
+
"agent_resume",
|
|
792
|
+
"human_input",
|
|
793
|
+
]
|
|
794
|
+
assert not only(rows, "error")
|
|
795
|
+
|
|
796
|
+
|
|
797
|
+
def test_the_two_interrupt_paths_do_not_double_emit(tmp_path, instrumented):
|
|
798
|
+
"""Both the lifecycle callback and the exception path fire; `Interrupt.id`
|
|
799
|
+
dedups them. A regression here doubles every pause in the dashboard."""
|
|
800
|
+
app = build_graph(checkpointer=InMemorySaver())
|
|
801
|
+
config = {"configurable": {"thread_id": "dedup"}}
|
|
802
|
+
app.invoke(empty_state(), config=config)
|
|
803
|
+
rows = read_events(tmp_path)
|
|
804
|
+
assert len(only(rows, "human_wait")) == 1
|
|
805
|
+
assert len(only(rows, "agent_pause")) == 1
|
|
806
|
+
|
|
807
|
+
|
|
808
|
+
# ---------------------------------------------------------------------------
|
|
809
|
+
# Failures
|
|
810
|
+
# ---------------------------------------------------------------------------
|
|
811
|
+
|
|
812
|
+
def test_a_node_failure_fails_the_agent_and_is_counted_once(tmp_path, instrumented):
|
|
813
|
+
def boom(state):
|
|
814
|
+
raise ValueError("node exploded")
|
|
815
|
+
|
|
816
|
+
app = build_simple([("boom", boom)], name="failing")
|
|
817
|
+
with pytest.raises(ValueError):
|
|
818
|
+
app.invoke(empty_state(), config={"configurable": {"thread_id": "fail"}})
|
|
819
|
+
|
|
820
|
+
rows = read_events(tmp_path)
|
|
821
|
+
hook = only(rows, "hook_completed")[0]
|
|
822
|
+
assert hook["outcome"] == "failed"
|
|
823
|
+
assert "node exploded" in hook["error"]
|
|
824
|
+
# `"failed"`, never `"failure"` — the server only counts
|
|
825
|
+
# error|failed|timeout|rejected as a failure.
|
|
826
|
+
assert only(rows, "agent_end")[0]["outcome"] == "failed"
|
|
827
|
+
# The hook already owns this failure. A standalone `error` event as well
|
|
828
|
+
# would make `sessionSummary.errorCount` report two failures for one
|
|
829
|
+
# exception, on every failed run.
|
|
830
|
+
assert not only(rows, "error")
|
|
831
|
+
|
|
832
|
+
|
|
833
|
+
def test_a_failure_no_span_owns_produces_exactly_one_error_event(tmp_path, instrumented):
|
|
834
|
+
from langgraph.errors import GraphRecursionError
|
|
835
|
+
|
|
836
|
+
model = fake_model(*[AIMessage("t%d" % i) for i in range(50)])
|
|
837
|
+
|
|
838
|
+
def plan(state):
|
|
839
|
+
model.invoke(state["messages"])
|
|
840
|
+
return {"n": state["n"] + 1, "vals": ["p"]}
|
|
841
|
+
|
|
842
|
+
graph = StateGraph(State)
|
|
843
|
+
graph.add_node("plan", plan)
|
|
844
|
+
graph.add_edge(START, "plan")
|
|
845
|
+
graph.add_conditional_edges("plan", lambda s: "plan", {"plan": "plan"})
|
|
846
|
+
app = graph.compile(name="runaway")
|
|
847
|
+
with pytest.raises(GraphRecursionError):
|
|
848
|
+
app.invoke(
|
|
849
|
+
empty_state(),
|
|
850
|
+
config={"configurable": {"thread_id": "recursion"}, "recursion_limit": 4},
|
|
851
|
+
)
|
|
852
|
+
|
|
853
|
+
rows = read_events(tmp_path)
|
|
854
|
+
# The recursion limit is enforced by the Pregel loop, not by a node, so no
|
|
855
|
+
# leaf reported it: this is exactly the case where a standalone `error`
|
|
856
|
+
# event is the only way the failure reaches the Errors surface.
|
|
857
|
+
assert len(only(rows, "error")) == 1
|
|
858
|
+
assert only(rows, "error")[0]["error_type"] == "GraphRecursionError"
|
|
859
|
+
assert only(rows, "agent_end")[0]["outcome"] == "failed"
|
|
860
|
+
# Strictly before agent_end: the graph closes the agent span at agent_end,
|
|
861
|
+
# so an error after it is attributed to nothing.
|
|
862
|
+
assert types_of(rows)[-2:] == ["error", "agent_end"]
|
|
863
|
+
|
|
864
|
+
|
|
865
|
+
def test_a_translator_that_raises_on_every_call_cannot_break_the_graph(
|
|
866
|
+
tmp_path, monkeypatch, instrumented, sink
|
|
867
|
+
):
|
|
868
|
+
sink.allow = True # the whole point is that the failures are logged, not raised
|
|
869
|
+
|
|
870
|
+
def explode(*args, **kwargs):
|
|
871
|
+
raise RuntimeError("translator is broken")
|
|
872
|
+
|
|
873
|
+
for name in ("_on_start", "_on_end", "_stash", "_stash_error", "_stash_messages"):
|
|
874
|
+
monkeypatch.setattr(adapter, name, explode)
|
|
875
|
+
|
|
876
|
+
model = fake_model(tool_calling_message())
|
|
877
|
+
|
|
878
|
+
def plan(state):
|
|
879
|
+
return {"messages": [model.invoke(state["messages"])], "vals": ["p"]}
|
|
880
|
+
|
|
881
|
+
def act(state):
|
|
882
|
+
return {"vals": [adder.invoke({"a": 2, "b": 3})]}
|
|
883
|
+
|
|
884
|
+
app = build_simple([("plan", plan), ("act", act)], name="broken")
|
|
885
|
+
result = app.invoke(empty_state(), config={"configurable": {"thread_id": "broken"}})
|
|
886
|
+
|
|
887
|
+
# The customer's run is untouched: right answer, no exception, no missing work.
|
|
888
|
+
assert result["vals"] == ["p", 5]
|
|
889
|
+
assert result["messages"][-1].content == "planned"
|
|
890
|
+
assert sink.records
|
|
891
|
+
|
|
892
|
+
|
|
893
|
+
def test_the_translators_swallow_by_policy_not_by_accident(monkeypatch, sink):
|
|
894
|
+
"""`FAILPROOFAI_SDK_STRICT=1` is what makes the failure policy testable.
|
|
895
|
+
|
|
896
|
+
Without it you can only ever prove "the customer's run still worked", never
|
|
897
|
+
"we swallowed the right thing" — and an adapter that swallowed
|
|
898
|
+
`BaseException` would pass the first check while silently breaking
|
|
899
|
+
cancellation in every async application that installed it.
|
|
900
|
+
"""
|
|
901
|
+
sink.allow = True # `safe()` logs the swallow, which is the whole design
|
|
902
|
+
assert getattr(adapter._on_start, "__failproofai_safe__", False)
|
|
903
|
+
assert getattr(adapter._on_end, "__failproofai_safe__", False)
|
|
904
|
+
|
|
905
|
+
def explode(_run):
|
|
906
|
+
raise RuntimeError("boom")
|
|
907
|
+
|
|
908
|
+
guarded = _core.safe(explode)
|
|
909
|
+
guarded(object()) # swallowed by default
|
|
910
|
+
|
|
911
|
+
monkeypatch.setenv("FAILPROOFAI_SDK_STRICT", "1")
|
|
912
|
+
_core.set_strict(None)
|
|
913
|
+
with pytest.raises(RuntimeError):
|
|
914
|
+
guarded(object())
|
|
915
|
+
|
|
916
|
+
def cancel(_run):
|
|
917
|
+
raise KeyboardInterrupt
|
|
918
|
+
|
|
919
|
+
_core.set_strict(False)
|
|
920
|
+
# A BaseException is never swallowed, strict or not.
|
|
921
|
+
with pytest.raises(KeyboardInterrupt):
|
|
922
|
+
_core.safe(cancel)(object())
|
|
923
|
+
|
|
924
|
+
|
|
925
|
+
def test_strict_mode_actually_surfaces_through_a_real_graph(monkeypatch, sink, instrumented):
|
|
926
|
+
"""The check above proves `safe()` re-raises. It does NOT prove the caller
|
|
927
|
+
ever sees it — and for a while, they didn't.
|
|
928
|
+
|
|
929
|
+
LangChain's `handle_event` catches every handler exception and logs
|
|
930
|
+
"Error in <handler>.<callback> callback" unless the handler sets
|
|
931
|
+
`raise_error`. We hard-coded that False, so under `FAILPROOFAI_SDK_STRICT=1`
|
|
932
|
+
`safe()` re-raised straight into LangChain's firewall and the fault was
|
|
933
|
+
swallowed one layer further out: the escape hatch silently did nothing on
|
|
934
|
+
the adapter people are most likely to be debugging. `raise_error` now
|
|
935
|
+
follows strict mode.
|
|
936
|
+
|
|
937
|
+
Note this asserts through a real `graph.invoke`, not against `safe()`. The
|
|
938
|
+
isolated test above passed the entire time the feature was broken.
|
|
939
|
+
"""
|
|
940
|
+
sink.allow = True
|
|
941
|
+
|
|
942
|
+
graph = StateGraph(State)
|
|
943
|
+
graph.add_node("bump", lambda state: {"vals": ["bumped"]})
|
|
944
|
+
graph.add_edge(START, "bump")
|
|
945
|
+
graph.add_edge("bump", END)
|
|
946
|
+
compiled = graph.compile(name="strict_probe")
|
|
947
|
+
|
|
948
|
+
def explode(self, *args, **kwargs):
|
|
949
|
+
raise RuntimeError("translator exploded")
|
|
950
|
+
|
|
951
|
+
monkeypatch.setattr(_core.RunTracker, "emit", explode)
|
|
952
|
+
|
|
953
|
+
# Default: the customer's graph is untouched and still returns the answer.
|
|
954
|
+
_core.set_strict(False)
|
|
955
|
+
assert compiled.invoke({"vals": [], "messages": []})["vals"] == ["bumped"]
|
|
956
|
+
|
|
957
|
+
# Strict: the fault reaches the caller instead of vanishing into a log line.
|
|
958
|
+
monkeypatch.setenv("FAILPROOFAI_SDK_STRICT", "1")
|
|
959
|
+
_core.set_strict(None)
|
|
960
|
+
with pytest.raises(RuntimeError, match="translator exploded"):
|
|
961
|
+
compiled.invoke({"vals": [], "messages": []})
|
|
962
|
+
|
|
963
|
+
|
|
964
|
+
def test_two_graphs_on_two_threads_never_mix_sessions(tmp_path, instrumented):
|
|
965
|
+
"""All adapter state is module-global — it has to be, because the configure
|
|
966
|
+
hook builds a fresh handler per callback manager. So the isolation has to
|
|
967
|
+
come from the run-id keys, and that is what this proves."""
|
|
968
|
+
import threading
|
|
969
|
+
|
|
970
|
+
app = build_simple([("n", lambda s: {"vals": ["x"]})], name="threaded")
|
|
971
|
+
barrier = threading.Barrier(4)
|
|
972
|
+
|
|
973
|
+
def worker(name):
|
|
974
|
+
barrier.wait(timeout=10)
|
|
975
|
+
for _ in range(3):
|
|
976
|
+
app.invoke(empty_state(), config={"configurable": {"thread_id": name}})
|
|
977
|
+
|
|
978
|
+
threads = [threading.Thread(target=worker, args=("t%d" % i,)) for i in range(4)]
|
|
979
|
+
for thread in threads:
|
|
980
|
+
thread.start()
|
|
981
|
+
for thread in threads:
|
|
982
|
+
thread.join(timeout=30)
|
|
983
|
+
|
|
984
|
+
rows = read_events(tmp_path)
|
|
985
|
+
assert {r["session_id"] for r in rows} == {"t0", "t1", "t2", "t3"}
|
|
986
|
+
per_session = {}
|
|
987
|
+
for row in rows:
|
|
988
|
+
per_session.setdefault(row["session_id"], []).append(row["type"])
|
|
989
|
+
for session, kinds in per_session.items():
|
|
990
|
+
assert kinds.count("agent_start") == 3, (session, kinds)
|
|
991
|
+
assert kinds.count("agent_end") == 3, (session, kinds)
|
|
992
|
+
assert kinds.count("hook_triggered") == 3, (session, kinds)
|
|
993
|
+
|
|
994
|
+
|
|
995
|
+
# ---------------------------------------------------------------------------
|
|
996
|
+
# Session resolution and interop
|
|
997
|
+
# ---------------------------------------------------------------------------
|
|
998
|
+
|
|
999
|
+
def test_session_id_prefers_the_documented_metadata_key_over_thread_id(
|
|
1000
|
+
tmp_path, instrumented
|
|
1001
|
+
):
|
|
1002
|
+
build_simple([("n", lambda s: {"vals": ["x"]})], name="s").invoke(
|
|
1003
|
+
empty_state(),
|
|
1004
|
+
config={
|
|
1005
|
+
"configurable": {"thread_id": "the-thread"},
|
|
1006
|
+
"metadata": {adapter.SESSION_METADATA_KEY: "chosen"},
|
|
1007
|
+
},
|
|
1008
|
+
)
|
|
1009
|
+
assert {r["session_id"] for r in read_events(tmp_path)} == {"chosen"}
|
|
1010
|
+
|
|
1011
|
+
|
|
1012
|
+
def test_session_id_falls_back_to_thread_id(tmp_path, instrumented):
|
|
1013
|
+
"""Verified against langgraph 1.2.10, contra the widely-reported claim that
|
|
1014
|
+
`thread_id` is no longer visible to callbacks: langchain-core's
|
|
1015
|
+
`ensure_config` stopped promoting it, and langgraph's `_PROPAGATE_TO_METADATA`
|
|
1016
|
+
puts it back."""
|
|
1017
|
+
build_simple([("n", lambda s: {"vals": ["x"]})], name="s").invoke(
|
|
1018
|
+
empty_state(), config={"configurable": {"thread_id": "from-thread"}}
|
|
1019
|
+
)
|
|
1020
|
+
assert {r["session_id"] for r in read_events(tmp_path)} == {"from-thread"}
|
|
1021
|
+
|
|
1022
|
+
|
|
1023
|
+
def test_session_id_falls_back_to_the_root_run_id(tmp_path, instrumented):
|
|
1024
|
+
build_simple([("n", lambda s: {"vals": ["x"]})], name="s").invoke(empty_state())
|
|
1025
|
+
sessions = {r["session_id"] for r in read_events(tmp_path)}
|
|
1026
|
+
assert len(sessions) == 1
|
|
1027
|
+
# A run id, not a synthesised one: a made-up session splits one run in two.
|
|
1028
|
+
assert _looks_like_a_uuid(next(iter(sessions)))
|
|
1029
|
+
|
|
1030
|
+
|
|
1031
|
+
def test_an_ambient_agent_scope_and_the_adapter_produce_one_tree(tmp_path, instrumented):
|
|
1032
|
+
with failproofai_sdk.agent("planner", goal="do the thing"):
|
|
1033
|
+
build_simple([("n", lambda s: {"vals": ["x"]})], name="inner").invoke(
|
|
1034
|
+
empty_state(), config={"configurable": {"thread_id": "ignored"}}
|
|
1035
|
+
)
|
|
1036
|
+
rows = read_events(tmp_path)
|
|
1037
|
+
assert len({r["session_id"] for r in rows}) == 1
|
|
1038
|
+
starts = only(rows, "agent_start")
|
|
1039
|
+
assert [r["agent_id"] for r in starts] == ["planner", "inner"]
|
|
1040
|
+
# This is the whole interop story: a hand-written outer bracket and an
|
|
1041
|
+
# adapter must produce one tree, not two disconnected sessions.
|
|
1042
|
+
assert starts[1]["parent_id"] == "planner"
|
|
1043
|
+
|
|
1044
|
+
|
|
1045
|
+
def test_an_explicit_session_id_option_wins(tmp_path):
|
|
1046
|
+
failproofai_sdk.instrument("langchain", session_id="forced")
|
|
1047
|
+
try:
|
|
1048
|
+
build_simple([("n", lambda s: {"vals": ["x"]})], name="s").invoke(
|
|
1049
|
+
empty_state(), config={"configurable": {"thread_id": "ignored"}}
|
|
1050
|
+
)
|
|
1051
|
+
finally:
|
|
1052
|
+
failproofai_sdk.uninstrument("langchain")
|
|
1053
|
+
assert {r["session_id"] for r in read_events(tmp_path)} == {"forced"}
|
|
1054
|
+
|
|
1055
|
+
|
|
1056
|
+
# ---------------------------------------------------------------------------
|
|
1057
|
+
# Install / uninstall
|
|
1058
|
+
# ---------------------------------------------------------------------------
|
|
1059
|
+
|
|
1060
|
+
def test_uninstrument_stops_recording_and_is_idempotent(tmp_path):
|
|
1061
|
+
failproofai_sdk.instrument("langchain")
|
|
1062
|
+
app = build_simple([("n", lambda s: {"vals": ["x"]})], name="s")
|
|
1063
|
+
app.invoke(empty_state(), config={"configurable": {"thread_id": "on"}})
|
|
1064
|
+
before = len(read_events(tmp_path))
|
|
1065
|
+
assert before
|
|
1066
|
+
|
|
1067
|
+
assert failproofai_sdk.uninstrument("langchain") == ("langchain",)
|
|
1068
|
+
assert failproofai_sdk.uninstrument("langchain") == ()
|
|
1069
|
+
assert adapter.ENV_VAR not in os.environ
|
|
1070
|
+
|
|
1071
|
+
app.invoke(empty_state(), config={"configurable": {"thread_id": "off"}})
|
|
1072
|
+
assert len(read_events(tmp_path)) == before
|
|
1073
|
+
|
|
1074
|
+
|
|
1075
|
+
def test_instrumenting_twice_does_not_double_record(tmp_path):
|
|
1076
|
+
failproofai_sdk.instrument("langchain")
|
|
1077
|
+
assert failproofai_sdk.instrument("langchain") == ()
|
|
1078
|
+
try:
|
|
1079
|
+
build_simple([("n", lambda s: {"vals": ["x"]})], name="s").invoke(
|
|
1080
|
+
empty_state(), config={"configurable": {"thread_id": "twice"}}
|
|
1081
|
+
)
|
|
1082
|
+
finally:
|
|
1083
|
+
failproofai_sdk.uninstrument("langchain")
|
|
1084
|
+
rows = read_events(tmp_path)
|
|
1085
|
+
assert len(only(rows, "agent_start")) == 1
|
|
1086
|
+
assert len(only(rows, "hook_triggered")) == 1
|
|
1087
|
+
|
|
1088
|
+
|
|
1089
|
+
def test_autodetect_picks_up_langchain(tmp_path):
|
|
1090
|
+
installed = failproofai_sdk.instrument()
|
|
1091
|
+
try:
|
|
1092
|
+
assert "langchain" in installed
|
|
1093
|
+
finally:
|
|
1094
|
+
failproofai_sdk.uninstrument()
|
|
1095
|
+
|
|
1096
|
+
|
|
1097
|
+
def test_open_leaves_are_closed_when_the_root_run_ends(tmp_path, instrumented):
|
|
1098
|
+
"""`agent_end` force-closes open pauses but not tools, models or humans."""
|
|
1099
|
+
from failproofai_sdk.integrations.langchain import _RunInfo, _STATE
|
|
1100
|
+
|
|
1101
|
+
def leaky(state):
|
|
1102
|
+
# Simulate a framework that never delivered the end callback: register a
|
|
1103
|
+
# tool run by hand and leave it open.
|
|
1104
|
+
with _STATE.lock:
|
|
1105
|
+
root = next(i for i in _STATE.runs.values() if i.kind == "root")
|
|
1106
|
+
orphan = _RunInfo(
|
|
1107
|
+
id="orphan", parent=root.id, name="ghost", run_type="tool", kind="tool"
|
|
1108
|
+
)
|
|
1109
|
+
orphan.root = root.id
|
|
1110
|
+
orphan.session = root.session
|
|
1111
|
+
orphan.tool_call_id = "orphan"
|
|
1112
|
+
_STATE.runs["orphan"] = orphan
|
|
1113
|
+
_STATE.tracker.link("orphan", root.id)
|
|
1114
|
+
_STATE.tracker.emit(
|
|
1115
|
+
"tool_use", "orphan", parent_key=root.id, tool_name="ghost", tool_call_id="orphan"
|
|
1116
|
+
)
|
|
1117
|
+
return {"vals": ["x"]}
|
|
1118
|
+
|
|
1119
|
+
build_simple([("leaky", leaky)], name="leaky").invoke(
|
|
1120
|
+
empty_state(), config={"configurable": {"thread_id": "leak"}}
|
|
1121
|
+
)
|
|
1122
|
+
rows = read_events(tmp_path)
|
|
1123
|
+
ghosts = [r for r in only(rows, "tool_result") if r["tool_name"] == "ghost"]
|
|
1124
|
+
assert len(ghosts) == 1
|
|
1125
|
+
assert ghosts[0]["fw_incomplete"] is True
|
|
1126
|
+
assert rows[-1]["type"] == "agent_end"
|
|
1127
|
+
|
|
1128
|
+
|
|
1129
|
+
# ---------------------------------------------------------------------------
|
|
1130
|
+
# Async
|
|
1131
|
+
# ---------------------------------------------------------------------------
|
|
1132
|
+
|
|
1133
|
+
@pytest.mark.asyncio
|
|
1134
|
+
async def test_ainvoke_produces_the_same_shape(tmp_path, instrumented):
|
|
1135
|
+
model = fake_model(tool_calling_message())
|
|
1136
|
+
|
|
1137
|
+
async def plan(state, config: RunnableConfig):
|
|
1138
|
+
return {
|
|
1139
|
+
"messages": [await model.ainvoke(state["messages"], config=config)],
|
|
1140
|
+
"vals": ["p"],
|
|
1141
|
+
}
|
|
1142
|
+
|
|
1143
|
+
async def act(state, config: RunnableConfig):
|
|
1144
|
+
return {"vals": [await adder.ainvoke({"a": 1, "b": 2}, config=config)]}
|
|
1145
|
+
|
|
1146
|
+
app = build_simple([("plan", plan), ("act", act)], name="async_pipeline")
|
|
1147
|
+
await app.ainvoke(empty_state(), config={"configurable": {"thread_id": "async"}})
|
|
1148
|
+
|
|
1149
|
+
assert types_of(read_events(tmp_path)) == [
|
|
1150
|
+
"agent_start",
|
|
1151
|
+
"hook_triggered",
|
|
1152
|
+
"model_request",
|
|
1153
|
+
"model_response",
|
|
1154
|
+
"hook_completed",
|
|
1155
|
+
"hook_triggered",
|
|
1156
|
+
"tool_use",
|
|
1157
|
+
"tool_result",
|
|
1158
|
+
"hook_completed",
|
|
1159
|
+
"agent_end",
|
|
1160
|
+
]
|
|
1161
|
+
|
|
1162
|
+
|
|
1163
|
+
# ---------------------------------------------------------------------------
|
|
1164
|
+
# Structural anti-drift — the highest-value tests in this file
|
|
1165
|
+
# ---------------------------------------------------------------------------
|
|
1166
|
+
|
|
1167
|
+
def test_every_override_still_exists_on_its_framework_base():
|
|
1168
|
+
"""Reflect over the handler and check it still overrides something real.
|
|
1169
|
+
|
|
1170
|
+
This is the test that catches the failure mode nothing else can. If
|
|
1171
|
+
LangChain renames `on_tool_error` or drops `_start_trace`, our method stops
|
|
1172
|
+
being called, we emit nothing for that path, and every behavioural test in
|
|
1173
|
+
this file still passes — they assert on the events we *did* emit, and a
|
|
1174
|
+
silent gap looks exactly like a run that had no tool errors. The in-repo
|
|
1175
|
+
precedent is `cli/agenteye_cli/_click_compat.py`, where an `isinstance`
|
|
1176
|
+
check went quietly always-False while its own anti-drift test stayed green.
|
|
1177
|
+
"""
|
|
1178
|
+
cls = adapter.FailproofAITracer
|
|
1179
|
+
overrides = {
|
|
1180
|
+
name: fn
|
|
1181
|
+
for name, fn in vars(cls).items()
|
|
1182
|
+
if inspect.isfunction(fn) and not name.startswith("__")
|
|
1183
|
+
}
|
|
1184
|
+
assert len(overrides) >= 10, "reflection found nothing — the test would be vacuous"
|
|
1185
|
+
|
|
1186
|
+
for name, fn in sorted(overrides.items()):
|
|
1187
|
+
base = next((b for b in cls.__mro__[1:] if name in vars(b)), None)
|
|
1188
|
+
assert base is not None, (
|
|
1189
|
+
"%s overrides nothing on any base — it is dead code that the "
|
|
1190
|
+
"framework will never call" % name
|
|
1191
|
+
)
|
|
1192
|
+
base_fn = vars(base)[name]
|
|
1193
|
+
base_params = inspect.signature(base_fn).parameters
|
|
1194
|
+
for param_name, param in inspect.signature(fn).parameters.items():
|
|
1195
|
+
if param.kind in (param.VAR_KEYWORD, param.VAR_POSITIONAL):
|
|
1196
|
+
continue
|
|
1197
|
+
# Deliberately no "...or the base takes **kwargs" escape hatch: a
|
|
1198
|
+
# renamed parameter would still be swallowed by the base's **kwargs
|
|
1199
|
+
# and our own parameter would silently never be filled.
|
|
1200
|
+
assert param_name in base_params, (
|
|
1201
|
+
"%s.%s declares %r, which no longer appears in %s.%s%s"
|
|
1202
|
+
% (cls.__name__, name, param_name, base.__name__, name,
|
|
1203
|
+
inspect.signature(base_fn))
|
|
1204
|
+
)
|
|
1205
|
+
|
|
1206
|
+
|
|
1207
|
+
def test_the_graph_lifecycle_overrides_are_bound_to_the_real_langgraph_base():
|
|
1208
|
+
"""`on_interrupt` lives on a stand-in when langgraph is too old.
|
|
1209
|
+
|
|
1210
|
+
If the import silently fell back on a machine that *has* langgraph 1.2,
|
|
1211
|
+
langgraph's `isinstance(h, GraphCallbackHandler)` filter would never match
|
|
1212
|
+
us and HITL would go dark with nothing to show for it.
|
|
1213
|
+
"""
|
|
1214
|
+
from langgraph.callbacks import GraphCallbackHandler
|
|
1215
|
+
|
|
1216
|
+
assert adapter._GraphCallbackHandler is GraphCallbackHandler
|
|
1217
|
+
assert issubclass(adapter.FailproofAITracer, GraphCallbackHandler)
|
|
1218
|
+
for name in ("on_interrupt", "on_resume"):
|
|
1219
|
+
assert name in vars(GraphCallbackHandler)
|
|
1220
|
+
|
|
1221
|
+
|
|
1222
|
+
def test_the_handler_flags_the_callback_manager_depends_on():
|
|
1223
|
+
cls = adapter.FailproofAITracer
|
|
1224
|
+
# AsyncCallbackManager dispatches sync handlers through run_in_executor
|
|
1225
|
+
# unless run_inline, and that hop can REORDER callbacks — which scrambles
|
|
1226
|
+
# timestamp order and breaks every pairing in this file.
|
|
1227
|
+
assert cls.run_inline is True
|
|
1228
|
+
|
|
1229
|
+
# raise_error follows FAILPROOFAI_SDK_STRICT rather than being a constant, so it
|
|
1230
|
+
# has to be read off an INSTANCE -- which is also how LangChain reads it.
|
|
1231
|
+
# Normally False: LangChain already firewalls handler exceptions, and
|
|
1232
|
+
# raising would take the customer's graph down with our bug. Under strict
|
|
1233
|
+
# it must be True, or LangChain's firewall swallows the exception `safe()`
|
|
1234
|
+
# re-raises and the escape hatch does nothing at all here.
|
|
1235
|
+
handler = cls()
|
|
1236
|
+
_core.set_strict(False)
|
|
1237
|
+
assert handler.raise_error is False
|
|
1238
|
+
_core.set_strict(True)
|
|
1239
|
+
assert handler.raise_error is True
|
|
1240
|
+
_core.set_strict(None)
|
|
1241
|
+
|
|
1242
|
+
from langchain_core.callbacks.base import BaseCallbackHandler
|
|
1243
|
+
|
|
1244
|
+
assert hasattr(BaseCallbackHandler, "run_inline")
|
|
1245
|
+
assert hasattr(BaseCallbackHandler, "raise_error")
|
|
1246
|
+
|
|
1247
|
+
|
|
1248
|
+
def test_the_configure_hook_api_is_still_the_one_we_build_on():
|
|
1249
|
+
from langchain_core.tracers import context
|
|
1250
|
+
|
|
1251
|
+
assert callable(context.register_configure_hook)
|
|
1252
|
+
params = list(inspect.signature(context.register_configure_hook).parameters)
|
|
1253
|
+
assert params[:4] == ["context_var", "inheritable", "handle_class", "env_var"]
|
|
1254
|
+
# There is deliberately no deregister API — uninstall() has to work by
|
|
1255
|
+
# emptying the ContextVar and the env var instead. If one ever appears, this
|
|
1256
|
+
# assertion is the reminder to use it.
|
|
1257
|
+
assert not hasattr(context, "unregister_configure_hook")
|
|
1258
|
+
|
|
1259
|
+
|
|
1260
|
+
def test_the_control_flow_exception_hierarchy_is_still_where_we_look():
|
|
1261
|
+
from langgraph.errors import GraphBubbleUp, GraphInterrupt, ParentCommand
|
|
1262
|
+
|
|
1263
|
+
assert adapter._GraphBubbleUp is GraphBubbleUp
|
|
1264
|
+
assert issubclass(GraphInterrupt, GraphBubbleUp)
|
|
1265
|
+
assert issubclass(ParentCommand, GraphBubbleUp)
|
|
1266
|
+
# ...and the payload shape `_interrupts_of` reads.
|
|
1267
|
+
from langgraph.types import Interrupt
|
|
1268
|
+
|
|
1269
|
+
interrupt_obj = Interrupt(value={"prompt": "p"}, id="abc")
|
|
1270
|
+
assert adapter._interrupts_of(GraphInterrupt([interrupt_obj])) == (interrupt_obj,)
|
|
1271
|
+
# A ParentCommand is control flow but carries a Command, not interrupts, so
|
|
1272
|
+
# it must not be mistaken for a human pause.
|
|
1273
|
+
assert adapter._is_control_flow(ParentCommand(Command(resume=1)))
|
|
1274
|
+
assert adapter._interrupts_of(ParentCommand(Command(resume=1))) == ()
|
|
1275
|
+
assert not adapter._is_control_flow(ValueError("real"))
|
|
1276
|
+
|
|
1277
|
+
|
|
1278
|
+
def test_the_lifecycle_event_dataclasses_still_carry_the_fields_we_read():
|
|
1279
|
+
import dataclasses
|
|
1280
|
+
|
|
1281
|
+
from langgraph.callbacks import GraphInterruptEvent, GraphResumeEvent
|
|
1282
|
+
|
|
1283
|
+
interrupt_fields = {f.name for f in dataclasses.fields(GraphInterruptEvent)}
|
|
1284
|
+
assert {"run_id", "interrupts"} <= interrupt_fields
|
|
1285
|
+
assert "run_id" in {f.name for f in dataclasses.fields(GraphResumeEvent)}
|
|
1286
|
+
|
|
1287
|
+
from langgraph.types import Interrupt
|
|
1288
|
+
|
|
1289
|
+
assert {"value", "id"} <= {f.name for f in dataclasses.fields(Interrupt)}
|
|
1290
|
+
|
|
1291
|
+
|
|
1292
|
+
def test_the_node_filter_still_matches_real_langgraph_metadata(tmp_path):
|
|
1293
|
+
"""The node filter is a string comparison. Prove the strings still exist.
|
|
1294
|
+
|
|
1295
|
+
`metadata["langgraph_node"] == run.name` is what separates a node from every
|
|
1296
|
+
inner Runnable that inherits the same metadata. If the key is renamed, every
|
|
1297
|
+
node stops being a hook and the only symptom is a shorter timeline.
|
|
1298
|
+
"""
|
|
1299
|
+
from langchain_core.tracers.base import BaseTracer
|
|
1300
|
+
|
|
1301
|
+
seen = []
|
|
1302
|
+
|
|
1303
|
+
class Probe(BaseTracer):
|
|
1304
|
+
run_inline = True
|
|
1305
|
+
|
|
1306
|
+
def _persist_run(self, run):
|
|
1307
|
+
pass
|
|
1308
|
+
|
|
1309
|
+
def _start_trace(self, run):
|
|
1310
|
+
super()._start_trace(run)
|
|
1311
|
+
seen.append((run.name, dict(run.metadata or {}), run.run_type))
|
|
1312
|
+
|
|
1313
|
+
graph = StateGraph(State)
|
|
1314
|
+
graph.add_node("only_node", lambda s: {"vals": ["x"]})
|
|
1315
|
+
graph.add_edge(START, "only_node")
|
|
1316
|
+
graph.add_edge("only_node", END)
|
|
1317
|
+
graph.compile(name="probe").invoke(
|
|
1318
|
+
empty_state(), config={"callbacks": [Probe()], "configurable": {"thread_id": "t"}}
|
|
1319
|
+
)
|
|
1320
|
+
|
|
1321
|
+
root_name, root_meta, _ = seen[0]
|
|
1322
|
+
node_name, node_meta, _ = seen[1]
|
|
1323
|
+
assert root_meta.get("langgraph_node") is None
|
|
1324
|
+
assert node_meta["langgraph_node"] == node_name == "only_node"
|
|
1325
|
+
assert "langgraph_checkpoint_ns" in node_meta
|
|
1326
|
+
assert node_meta["thread_id"] == "t"
|
|
1327
|
+
# And a subgraph node's checkpoint namespace is still `|`-separated, which
|
|
1328
|
+
# is how nested agents are derived without recognising a Pregel object.
|
|
1329
|
+
seen.clear()
|
|
1330
|
+
outer = StateGraph(State)
|
|
1331
|
+
outer.add_node("child", subgraph())
|
|
1332
|
+
outer.add_edge(START, "child")
|
|
1333
|
+
outer.add_edge("child", END)
|
|
1334
|
+
outer.compile(name="outer").invoke(
|
|
1335
|
+
empty_state(), config={"callbacks": [Probe()], "configurable": {"thread_id": "t2"}}
|
|
1336
|
+
)
|
|
1337
|
+
inner = [m for name, m, _ in seen if m.get("langgraph_node") == name == "sub_step"]
|
|
1338
|
+
assert inner and "|" in inner[0]["langgraph_checkpoint_ns"]
|
|
1339
|
+
|
|
1340
|
+
|
|
1341
|
+
def test_the_tool_call_id_still_reaches_the_run_object():
|
|
1342
|
+
"""`tool_call_id` is read off `run.extra`, where `_create_tool_run` parks
|
|
1343
|
+
the callback kwargs. If that stops happening we would silently fall back to
|
|
1344
|
+
the run id and every tool would stop lining up with the provider's logs."""
|
|
1345
|
+
from langchain_core.tracers.base import BaseTracer
|
|
1346
|
+
|
|
1347
|
+
seen = []
|
|
1348
|
+
|
|
1349
|
+
class Probe(BaseTracer):
|
|
1350
|
+
run_inline = True
|
|
1351
|
+
|
|
1352
|
+
def _persist_run(self, run):
|
|
1353
|
+
pass
|
|
1354
|
+
|
|
1355
|
+
def _start_trace(self, run):
|
|
1356
|
+
super()._start_trace(run)
|
|
1357
|
+
if run.run_type == "tool":
|
|
1358
|
+
seen.append(dict(run.extra or {}))
|
|
1359
|
+
|
|
1360
|
+
model = fake_model(tool_calling_message())
|
|
1361
|
+
|
|
1362
|
+
def plan(state):
|
|
1363
|
+
return {"messages": [model.invoke(state["messages"])], "vals": ["p"]}
|
|
1364
|
+
|
|
1365
|
+
graph = StateGraph(State)
|
|
1366
|
+
graph.add_node("plan", plan)
|
|
1367
|
+
graph.add_node("tools", ToolNode([adder]))
|
|
1368
|
+
graph.add_edge(START, "plan")
|
|
1369
|
+
graph.add_edge("plan", "tools")
|
|
1370
|
+
graph.add_edge("tools", END)
|
|
1371
|
+
graph.compile(name="ids").invoke(
|
|
1372
|
+
empty_state(), config={"callbacks": [Probe()], "configurable": {"thread_id": "t"}}
|
|
1373
|
+
)
|
|
1374
|
+
assert seen and seen[0].get("tool_call_id") == "call_abc"
|
|
1375
|
+
|
|
1376
|
+
|
|
1377
|
+
def test_usage_metadata_survives_to_on_llm_end():
|
|
1378
|
+
"""The primary token source. The two fallbacks exist because providers
|
|
1379
|
+
disagree; this asserts the primary is still the primary."""
|
|
1380
|
+
from langchain_core.tracers.base import BaseTracer
|
|
1381
|
+
|
|
1382
|
+
seen = []
|
|
1383
|
+
|
|
1384
|
+
class Probe(BaseTracer):
|
|
1385
|
+
run_inline = True
|
|
1386
|
+
|
|
1387
|
+
def _persist_run(self, run):
|
|
1388
|
+
pass
|
|
1389
|
+
|
|
1390
|
+
def on_llm_end(self, response, *, run_id, **kwargs):
|
|
1391
|
+
seen.append(adapter._usage(response))
|
|
1392
|
+
return super().on_llm_end(response, run_id=run_id, **kwargs)
|
|
1393
|
+
|
|
1394
|
+
model = fake_model(tool_calling_message())
|
|
1395
|
+
model.invoke([HumanMessage("hi")], config={"callbacks": [Probe()]})
|
|
1396
|
+
assert seen == [{"input_tokens": 11, "output_tokens": 5, "total_tokens": 16}]
|
|
1397
|
+
|
|
1398
|
+
|
|
1399
|
+
# ---------------------------------------------------------------------------
|
|
1400
|
+
# A root run that is itself a leaf
|
|
1401
|
+
# ---------------------------------------------------------------------------
|
|
1402
|
+
#
|
|
1403
|
+
# `ChatOpenAI(...).invoke(...)` outside any graph arrives as ONE run with no
|
|
1404
|
+
# parent and `run_type="chat_model"`. Handled only as a root it produced
|
|
1405
|
+
# `agent_start`/`agent_end` and nothing else: no `model_request`, no
|
|
1406
|
+
# `model_response`, so the model name, both token counts and the latency of a
|
|
1407
|
+
# direct model call were dropped while the trace still looked populated.
|
|
1408
|
+
#
|
|
1409
|
+
# Direct `.invoke()` is not an edge case — a classifier, a summariser and a
|
|
1410
|
+
# one-shot rewrite are all shaped exactly like this.
|
|
1411
|
+
|
|
1412
|
+
|
|
1413
|
+
def test_a_bare_model_call_still_emits_its_model_pair(tmp_path, instrumented):
|
|
1414
|
+
with failproofai_sdk.session():
|
|
1415
|
+
fake_model(AIMessage("hi", usage_metadata={
|
|
1416
|
+
"input_tokens": 7, "output_tokens": 3, "total_tokens": 10,
|
|
1417
|
+
})).invoke("say hi")
|
|
1418
|
+
|
|
1419
|
+
rows = read_events(tmp_path)
|
|
1420
|
+
kinds = types_of(rows)
|
|
1421
|
+
assert "model_request" in kinds, (
|
|
1422
|
+
f"a bare model call emitted {kinds} — the model pair is missing, so the "
|
|
1423
|
+
f"model name, token counts and latency of every direct .invoke() are lost"
|
|
1424
|
+
)
|
|
1425
|
+
assert "model_response" in kinds
|
|
1426
|
+
|
|
1427
|
+
|
|
1428
|
+
def test_a_bare_model_call_records_tokens_and_an_int_duration(tmp_path, instrumented):
|
|
1429
|
+
with failproofai_sdk.session():
|
|
1430
|
+
fake_model(AIMessage("hi", usage_metadata={
|
|
1431
|
+
"input_tokens": 7, "output_tokens": 3, "total_tokens": 10,
|
|
1432
|
+
})).invoke("say hi")
|
|
1433
|
+
|
|
1434
|
+
response = only(read_events(tmp_path), "model_response")[0]
|
|
1435
|
+
assert response["input_tokens"] == 7
|
|
1436
|
+
assert response["output_tokens"] == 3
|
|
1437
|
+
# u32 column: a float silently NULLs it server-side.
|
|
1438
|
+
assert isinstance(response["duration_ms"], int)
|
|
1439
|
+
assert response["model"]
|
|
1440
|
+
|
|
1441
|
+
|
|
1442
|
+
def test_a_bare_model_calls_pair_sits_inside_its_agent_span(tmp_path, instrumented):
|
|
1443
|
+
"""Order matters: the dashboard closes the agent span at `agent_end`, so a
|
|
1444
|
+
`model_response` after it is attributed to nothing."""
|
|
1445
|
+
with failproofai_sdk.session():
|
|
1446
|
+
fake_model(AIMessage("hi")).invoke("say hi")
|
|
1447
|
+
|
|
1448
|
+
kinds = types_of(read_events(tmp_path))
|
|
1449
|
+
assert kinds.index("agent_start") < kinds.index("model_request")
|
|
1450
|
+
assert kinds.index("model_response") < kinds.index("agent_end")
|
|
1451
|
+
|
|
1452
|
+
|
|
1453
|
+
def test_a_bare_model_pair_shares_one_request_id(tmp_path, instrumented):
|
|
1454
|
+
with failproofai_sdk.session():
|
|
1455
|
+
fake_model(AIMessage("hi")).invoke("say hi")
|
|
1456
|
+
|
|
1457
|
+
rows = read_events(tmp_path)
|
|
1458
|
+
request = only(rows, "model_request")[0]
|
|
1459
|
+
response = only(rows, "model_response")[0]
|
|
1460
|
+
assert request["request_id"] == response["request_id"]
|
|
1461
|
+
|
|
1462
|
+
|
|
1463
|
+
def test_a_graph_run_is_not_treated_as_a_leaf(tmp_path, instrumented):
|
|
1464
|
+
"""The fix is additive and must not fire for a chain-typed root."""
|
|
1465
|
+
build_simple([("only", lambda state: {"vals": ["x"]})]).invoke(empty_state())
|
|
1466
|
+
rows = read_events(tmp_path)
|
|
1467
|
+
# A graph root emits no model pair of its own — only its nodes do.
|
|
1468
|
+
assert types_of(rows).count("agent_start") == 1
|
|
1469
|
+
|
|
1470
|
+
|
|
1471
|
+
# ---------------------------------------------------------------------------
|
|
1472
|
+
# Two roots that merely OVERLAP are not a resume
|
|
1473
|
+
# ---------------------------------------------------------------------------
|
|
1474
|
+
#
|
|
1475
|
+
# `_start_root` reuses an existing session's agent when that agent is still
|
|
1476
|
+
# open, because that is what an interrupt/resume looks like: the paused
|
|
1477
|
+
# `.invoke()` deliberately did not close its agent and the resuming one must not
|
|
1478
|
+
# open a second root span for the same logical run.
|
|
1479
|
+
#
|
|
1480
|
+
# "Still open" is ALSO true of two roots that merely overlap in time under one
|
|
1481
|
+
# session id, and that is not exotic — langchain-core opens one root run **per
|
|
1482
|
+
# input** for `.batch()`, and any two requests carrying the same conversation id
|
|
1483
|
+
# through `SESSION_METADATA_KEY` do the same. Read as a resume, the second root
|
|
1484
|
+
# got no `agent_start` at all, its work was relabelled with the first root's
|
|
1485
|
+
# `agent_id`, the first root to finish closed the shared agent, and everything
|
|
1486
|
+
# the other root emitted afterwards resolved to nothing and was DROPPED.
|
|
1487
|
+
#
|
|
1488
|
+
# The discriminator is `open_pauses`: `_end_root` skips `agent_end` exactly when
|
|
1489
|
+
# it is non-empty, which is the only way an agent outlives its root, and
|
|
1490
|
+
# `_suspend` is the only thing that fills it.
|
|
1491
|
+
|
|
1492
|
+
|
|
1493
|
+
def test_two_overlapping_roots_in_one_session_are_two_agents(tmp_path, instrumented):
|
|
1494
|
+
import threading
|
|
1495
|
+
|
|
1496
|
+
# The barrier is the whole point: both roots are guaranteed to be OPEN at
|
|
1497
|
+
# the same time, which is the state that used to be misread as a resume.
|
|
1498
|
+
# Without it this races and passes against the bug about half the time.
|
|
1499
|
+
barrier = threading.Barrier(2, timeout=10)
|
|
1500
|
+
|
|
1501
|
+
def hold(state):
|
|
1502
|
+
barrier.wait()
|
|
1503
|
+
return {"vals": ["x"]}
|
|
1504
|
+
|
|
1505
|
+
app = build_simple([("n", hold)], name="overlap")
|
|
1506
|
+
|
|
1507
|
+
def run():
|
|
1508
|
+
app.invoke(
|
|
1509
|
+
empty_state(),
|
|
1510
|
+
config={"metadata": {adapter.SESSION_METADATA_KEY: "one-session"}},
|
|
1511
|
+
)
|
|
1512
|
+
|
|
1513
|
+
threads = [threading.Thread(target=run) for _ in range(2)]
|
|
1514
|
+
for thread in threads:
|
|
1515
|
+
thread.start()
|
|
1516
|
+
for thread in threads:
|
|
1517
|
+
thread.join(timeout=30)
|
|
1518
|
+
|
|
1519
|
+
rows = read_events(tmp_path)
|
|
1520
|
+
assert {r["session_id"] for r in rows} == {"one-session"}
|
|
1521
|
+
starts = only(rows, "agent_start")
|
|
1522
|
+
ends = only(rows, "agent_end")
|
|
1523
|
+
assert len(starts) == 2, (
|
|
1524
|
+
f"two overlapping roots produced {len(starts)} agent_start(s): the second "
|
|
1525
|
+
f"root was read as a resume of the first"
|
|
1526
|
+
)
|
|
1527
|
+
assert len(ends) == 2
|
|
1528
|
+
# Distinct runs, not one run reported twice.
|
|
1529
|
+
assert len({r["fw_run_id"] for r in starts}) == 2
|
|
1530
|
+
# Nothing was dropped on the way: each root ran the node once.
|
|
1531
|
+
assert types_of(rows).count("hook_triggered") == 2
|
|
1532
|
+
assert types_of(rows).count("hook_completed") == 2
|
|
1533
|
+
# `sink` (autouse) fails this test on the "could not resolve a session for
|
|
1534
|
+
# run ... and is dropping its events" warning the old behaviour produced,
|
|
1535
|
+
# which is the other half of the regression and the half that was silent.
|
|
1536
|
+
|
|
1537
|
+
|
|
1538
|
+
def test_a_genuine_interrupt_resume_is_still_one_agent_not_two(tmp_path, instrumented):
|
|
1539
|
+
"""The counterweight: `open_pauses` must not disable the resume path.
|
|
1540
|
+
|
|
1541
|
+
Deleting the resume branch would also "fix" the overlap bug above, at the
|
|
1542
|
+
cost of splitting every human approval into two root spans and zeroing the
|
|
1543
|
+
`agent_pause` -> `agent_resume` interval that is the only measure of how
|
|
1544
|
+
long the human took.
|
|
1545
|
+
"""
|
|
1546
|
+
app = build_graph(checkpointer=InMemorySaver())
|
|
1547
|
+
config = {"configurable": {"thread_id": "resume-one"}}
|
|
1548
|
+
app.invoke(empty_state(), config=config)
|
|
1549
|
+
app.invoke(Command(resume="yes"), config=config)
|
|
1550
|
+
|
|
1551
|
+
rows = read_events(tmp_path)
|
|
1552
|
+
roots = [r for r in only(rows, "agent_start") if not r.get("parent_id")]
|
|
1553
|
+
assert len(roots) == 1, "the resuming .invoke() opened a second root span"
|
|
1554
|
+
assert len([r for r in only(rows, "agent_end") if r["agent_id"] == "root_graph"]) == 1
|
|
1555
|
+
assert types_of(only(rows, "agent_pause", "agent_resume")) == [
|
|
1556
|
+
"agent_pause",
|
|
1557
|
+
"agent_resume",
|
|
1558
|
+
]
|
|
1559
|
+
|
|
1560
|
+
|
|
1561
|
+
# ---------------------------------------------------------------------------
|
|
1562
|
+
# A failing root-run-that-is-a-leaf owns its failure exactly once
|
|
1563
|
+
# ---------------------------------------------------------------------------
|
|
1564
|
+
#
|
|
1565
|
+
# `_on_end` returns straight after `_end_root` for a root, so the line at the
|
|
1566
|
+
# bottom of the function that marks the failure as owned by the span below never
|
|
1567
|
+
# ran for a root that was ALSO a leaf. The same exception was then reported
|
|
1568
|
+
# twice — once as `tool_result.error` / `model_response.error` and again as a
|
|
1569
|
+
# standalone `error` event — and the server derives `is_error` from both, so one
|
|
1570
|
+
# failure counted as two on `sessionSummary.errorCount`. The identical failure
|
|
1571
|
+
# one Runnable deeper counted as one.
|
|
1572
|
+
|
|
1573
|
+
|
|
1574
|
+
def test_a_failing_bare_tool_reports_its_error_once(tmp_path, instrumented):
|
|
1575
|
+
with failproofai_sdk.session():
|
|
1576
|
+
with pytest.raises(RuntimeError, match="tool boom"):
|
|
1577
|
+
exploder.invoke({"x": 1})
|
|
1578
|
+
|
|
1579
|
+
rows = read_events(tmp_path)
|
|
1580
|
+
result = only(rows, "tool_result")[0]
|
|
1581
|
+
assert "tool boom" in result["error"]
|
|
1582
|
+
# The span that owns the failure has reported it; a standalone `error` event
|
|
1583
|
+
# on top is the same failure counted twice.
|
|
1584
|
+
assert not only(rows, "error"), (
|
|
1585
|
+
"a top-level tool failure was reported both on tool_result and as a "
|
|
1586
|
+
"standalone error event"
|
|
1587
|
+
)
|
|
1588
|
+
assert only(rows, "agent_end")[0]["outcome"] == "failed"
|
|
1589
|
+
|
|
1590
|
+
|
|
1591
|
+
def test_a_failing_bare_model_call_reports_its_error_once(tmp_path, instrumented):
|
|
1592
|
+
class _BoomModel(GenericFakeChatModel):
|
|
1593
|
+
def _generate(self, *args, **kwargs):
|
|
1594
|
+
raise RuntimeError("model boom")
|
|
1595
|
+
|
|
1596
|
+
with failproofai_sdk.session():
|
|
1597
|
+
with pytest.raises(RuntimeError, match="model boom"):
|
|
1598
|
+
_BoomModel(messages=iter([])).invoke("say hi")
|
|
1599
|
+
|
|
1600
|
+
rows = read_events(tmp_path)
|
|
1601
|
+
response = only(rows, "model_response")[0]
|
|
1602
|
+
assert "model boom" in response["error"]
|
|
1603
|
+
assert response["stop_reason"] == "error"
|
|
1604
|
+
assert not only(rows, "error")
|
|
1605
|
+
assert only(rows, "agent_end")[0]["outcome"] == "failed"
|
|
1606
|
+
|
|
1607
|
+
|
|
1608
|
+
def test_a_failure_below_the_root_still_produces_its_one_error_event(
|
|
1609
|
+
tmp_path, instrumented
|
|
1610
|
+
):
|
|
1611
|
+
"""The other side of the same line: a root nothing below reported must still
|
|
1612
|
+
get exactly one standalone `error`, or the failure reaches no surface."""
|
|
1613
|
+
|
|
1614
|
+
def boom(_payload):
|
|
1615
|
+
raise RuntimeError("chain boom")
|
|
1616
|
+
|
|
1617
|
+
with failproofai_sdk.session():
|
|
1618
|
+
with pytest.raises(RuntimeError, match="chain boom"):
|
|
1619
|
+
RunnableLambda(boom).with_config(run_name="boomer").invoke({"x": 1})
|
|
1620
|
+
|
|
1621
|
+
rows = read_events(tmp_path)
|
|
1622
|
+
assert len(only(rows, "error")) == 1
|
|
1623
|
+
assert only(rows, "agent_end")[0]["outcome"] == "failed"
|
|
1624
|
+
|
|
1625
|
+
|
|
1626
|
+
# ---------------------------------------------------------------------------
|
|
1627
|
+
# `tool_result.output` is the tool's result, not a repr of the envelope
|
|
1628
|
+
# ---------------------------------------------------------------------------
|
|
1629
|
+
#
|
|
1630
|
+
# A tool handed the LLM's `ToolCall` dict — what `bind_tools` produces and what
|
|
1631
|
+
# every modern tool loop passes — returns a `ToolMessage`. `truncate` has no
|
|
1632
|
+
# JSON shape for one, so the single most-read field in a tool loop rendered as
|
|
1633
|
+
# `ToolMessage(content='3', name='adder', tool_call_id='call_zz', ...)`.
|
|
1634
|
+
#
|
|
1635
|
+
# `status` is the second half: a `ToolMessage` carries `status="error"` when the
|
|
1636
|
+
# tool failed but the framework turned the exception into a message for the
|
|
1637
|
+
# model instead of raising. `run.error` is empty on that path, so the failure
|
|
1638
|
+
# had no representation at all — `is_error` 0, a green span, and the text of the
|
|
1639
|
+
# failure sitting in an output field nobody filters on.
|
|
1640
|
+
|
|
1641
|
+
|
|
1642
|
+
def test_a_tool_called_with_a_tool_call_records_its_content_not_a_repr(
|
|
1643
|
+
tmp_path, instrumented
|
|
1644
|
+
):
|
|
1645
|
+
with failproofai_sdk.session():
|
|
1646
|
+
adder.invoke(
|
|
1647
|
+
{"name": "adder", "args": {"a": 1, "b": 2}, "id": "call_zz", "type": "tool_call"}
|
|
1648
|
+
)
|
|
1649
|
+
|
|
1650
|
+
result = only(read_events(tmp_path), "tool_result")[0]
|
|
1651
|
+
assert result["output"] == "3", (
|
|
1652
|
+
f"tool_result.output is {result['output']!r} — the ToolMessage envelope "
|
|
1653
|
+
f"leaked instead of the tool's own result"
|
|
1654
|
+
)
|
|
1655
|
+
assert "ToolMessage(" not in str(result["output"])
|
|
1656
|
+
assert result["tool_call_id"] == "call_zz"
|
|
1657
|
+
|
|
1658
|
+
|
|
1659
|
+
def test_a_tool_that_fails_without_raising_is_still_an_error(tmp_path, instrumented):
|
|
1660
|
+
@tool
|
|
1661
|
+
def quiet_failer(x: int) -> str:
|
|
1662
|
+
"""Fails without raising: returns an error-status ToolMessage."""
|
|
1663
|
+
return ToolMessage(content="upstream 503", tool_call_id="unused", status="error")
|
|
1664
|
+
|
|
1665
|
+
with failproofai_sdk.session():
|
|
1666
|
+
quiet_failer.invoke(
|
|
1667
|
+
{"name": "quiet_failer", "args": {"x": 1}, "id": "call_q", "type": "tool_call"}
|
|
1668
|
+
)
|
|
1669
|
+
|
|
1670
|
+
result = only(read_events(tmp_path), "tool_result")[0]
|
|
1671
|
+
assert result.get("error"), (
|
|
1672
|
+
"a tool that reported failure through ToolMessage(status='error') was "
|
|
1673
|
+
"recorded as a success"
|
|
1674
|
+
)
|
|
1675
|
+
assert "upstream 503" in result["error"]
|
|
1676
|
+
|
|
1677
|
+
|
|
1678
|
+
def test_a_successful_tool_message_carries_no_error(tmp_path, instrumented):
|
|
1679
|
+
"""The `status` read must not turn every ToolMessage into a failure."""
|
|
1680
|
+
with failproofai_sdk.session():
|
|
1681
|
+
adder.invoke(
|
|
1682
|
+
{"name": "adder", "args": {"a": 1, "b": 2}, "id": "call_ok", "type": "tool_call"}
|
|
1683
|
+
)
|
|
1684
|
+
assert only(read_events(tmp_path), "tool_result")[0].get("error") is None
|
|
1685
|
+
|
|
1686
|
+
|
|
1687
|
+
# ---------------------------------------------------------------------------
|
|
1688
|
+
# The `error` event does not repeat its own type
|
|
1689
|
+
# ---------------------------------------------------------------------------
|
|
1690
|
+
#
|
|
1691
|
+
# `error` is the one event that carries `error_type` as its OWN field, and the
|
|
1692
|
+
# server builds the row's `summary` as "<error_type>: <message>". Feeding it
|
|
1693
|
+
# `_error_text` — which prefixes the type because `tool_result.error` and
|
|
1694
|
+
# `agent_end.summary` have nowhere else to say it — rendered every entry on the
|
|
1695
|
+
# Errors surface as `ValueError: ValueError: denominator must be non-zero`.
|
|
1696
|
+
# CrewAI, LlamaIndex and Pydantic AI all pass a bare `str(exc)` here.
|
|
1697
|
+
|
|
1698
|
+
|
|
1699
|
+
def test_the_error_events_message_does_not_repeat_its_own_type(tmp_path, instrumented):
|
|
1700
|
+
def boom(_payload):
|
|
1701
|
+
raise RuntimeError("chain boom")
|
|
1702
|
+
|
|
1703
|
+
with failproofai_sdk.session():
|
|
1704
|
+
with pytest.raises(RuntimeError, match="chain boom"):
|
|
1705
|
+
RunnableLambda(boom).with_config(run_name="boomer").invoke({"x": 1})
|
|
1706
|
+
|
|
1707
|
+
rows = read_events(tmp_path)
|
|
1708
|
+
event = only(rows, "error")[0]
|
|
1709
|
+
assert event["error_type"] == "RuntimeError"
|
|
1710
|
+
assert event["message"] == "chain boom", (
|
|
1711
|
+
f"message is {event['message']!r} — the server renders summary as "
|
|
1712
|
+
f"'<error_type>: <message>', so a prefixed message says it twice"
|
|
1713
|
+
)
|
|
1714
|
+
# Scoped to the `error` event: `agent_end.summary` has no `error_type`
|
|
1715
|
+
# field beside it, so it keeps naming the exception type itself.
|
|
1716
|
+
assert only(rows, "agent_end")[0]["summary"] == "RuntimeError: chain boom"
|
|
1717
|
+
|
|
1718
|
+
|
|
1719
|
+
# ---------------------------------------------------------------------------
|
|
1720
|
+
# `uninstrument()` when the trace env var was exported by somebody else
|
|
1721
|
+
# ---------------------------------------------------------------------------
|
|
1722
|
+
#
|
|
1723
|
+
# A configure hook cannot be deregistered, so removal is "make the hook produce
|
|
1724
|
+
# nothing" — and neither of the two levers `uninstall()` had actually does that
|
|
1725
|
+
# in every process. Clearing `_HANDLER_VAR` only reaches contexts derived from
|
|
1726
|
+
# the caller's, and the env var is unset only when `install()` was the one that
|
|
1727
|
+
# set it (it must not clobber somebody else's environment). Exported by a
|
|
1728
|
+
# Dockerfile or a CI job, it left `_configure` constructing a live zero-arg
|
|
1729
|
+
# tracer per callback manager, and a fully torn-down adapter went on recording
|
|
1730
|
+
# every event forever.
|
|
1731
|
+
|
|
1732
|
+
|
|
1733
|
+
def test_uninstrument_stops_recording_when_the_env_var_was_already_set(
|
|
1734
|
+
tmp_path, monkeypatch
|
|
1735
|
+
):
|
|
1736
|
+
monkeypatch.setenv(adapter.ENV_VAR, "1")
|
|
1737
|
+
failproofai_sdk.instrument("langchain")
|
|
1738
|
+
app = build_simple([("n", lambda s: {"vals": ["x"]})], name="s")
|
|
1739
|
+
app.invoke(empty_state(), config={"configurable": {"thread_id": "on"}})
|
|
1740
|
+
before = len(read_events(tmp_path))
|
|
1741
|
+
assert before
|
|
1742
|
+
|
|
1743
|
+
assert failproofai_sdk.uninstrument("langchain") == ("langchain",)
|
|
1744
|
+
# Deliberately still set: `install()` did not set it, so `uninstall()` does
|
|
1745
|
+
# not get to remove it. That is exactly why it cannot be the kill switch.
|
|
1746
|
+
assert os.environ.get(adapter.ENV_VAR) == "1"
|
|
1747
|
+
|
|
1748
|
+
app.invoke(empty_state(), config={"configurable": {"thread_id": "off"}})
|
|
1749
|
+
assert len(read_events(tmp_path)) == before, (
|
|
1750
|
+
"the adapter kept recording after uninstrument() because the trace env "
|
|
1751
|
+
"var was set before instrument() ran"
|
|
1752
|
+
)
|
|
1753
|
+
|
|
1754
|
+
|
|
1755
|
+
def test_reinstrumenting_after_that_teardown_records_again(tmp_path, monkeypatch):
|
|
1756
|
+
"""The kill switch must be a switch, not a one-way fuse."""
|
|
1757
|
+
monkeypatch.setenv(adapter.ENV_VAR, "1")
|
|
1758
|
+
failproofai_sdk.instrument("langchain")
|
|
1759
|
+
failproofai_sdk.uninstrument("langchain")
|
|
1760
|
+
failproofai_sdk.instrument("langchain")
|
|
1761
|
+
try:
|
|
1762
|
+
build_simple([("n", lambda s: {"vals": ["x"]})], name="s").invoke(
|
|
1763
|
+
empty_state(), config={"configurable": {"thread_id": "again"}}
|
|
1764
|
+
)
|
|
1765
|
+
finally:
|
|
1766
|
+
failproofai_sdk.uninstrument("langchain")
|
|
1767
|
+
assert types_of(read_events(tmp_path)).count("agent_start") == 1
|
|
1768
|
+
|
|
1769
|
+
|
|
1770
|
+
# ---------------------------------------------------------------------------
|
|
1771
|
+
# The node key and the run name are BOTH the user's to choose
|
|
1772
|
+
# ---------------------------------------------------------------------------
|
|
1773
|
+
#
|
|
1774
|
+
# `_node_of` matched on `run.name == metadata["langgraph_node"]` alone, and both
|
|
1775
|
+
# sides of that comparison are strings a user picks. Every inner run of a node
|
|
1776
|
+
# inherits `langgraph_node`, so the moment an inner run happens to carry the
|
|
1777
|
+
# node's name it was recorded as a second visit to the node instead of as what
|
|
1778
|
+
# it is — and the event that was actually worth having never got emitted.
|
|
1779
|
+
#
|
|
1780
|
+
# Verified against langgraph 1.2.11: whatever you hand `add_node`, the node's
|
|
1781
|
+
# OWN run is always a `chain` run tagged `graph:step:N`, and the thing you
|
|
1782
|
+
# passed runs beneath it tagged `seq:step:N`. Those two facts are the fix.
|
|
1783
|
+
|
|
1784
|
+
|
|
1785
|
+
def test_a_node_named_after_its_tool_still_records_the_tool(tmp_path, instrumented):
|
|
1786
|
+
"""`add_node("adder", ToolNode([adder]))` — the obvious naming — used to
|
|
1787
|
+
delete the tool call: no `tool_use`, no `tool_result`, no `tool_call_id`,
|
|
1788
|
+
and `/tools` showing the call had never happened."""
|
|
1789
|
+
|
|
1790
|
+
def plan(state):
|
|
1791
|
+
return {"messages": [tool_calling_message()], "vals": ["plan"]}
|
|
1792
|
+
|
|
1793
|
+
graph = StateGraph(State)
|
|
1794
|
+
graph.add_node("plan", plan)
|
|
1795
|
+
graph.add_node("adder", ToolNode([adder])) # node key == tool name
|
|
1796
|
+
graph.add_edge(START, "plan")
|
|
1797
|
+
graph.add_edge("plan", "adder")
|
|
1798
|
+
graph.add_edge("adder", END)
|
|
1799
|
+
graph.compile(name="collide").invoke(
|
|
1800
|
+
empty_state(), config={"configurable": {"thread_id": "tool-collision"}}
|
|
1801
|
+
)
|
|
1802
|
+
|
|
1803
|
+
rows = read_events(tmp_path)
|
|
1804
|
+
tools = only(rows, "tool_use", "tool_result")
|
|
1805
|
+
assert types_of(tools) == ["tool_use", "tool_result"], (
|
|
1806
|
+
"the tool run was misfiled as a second visit to the node of the same name"
|
|
1807
|
+
)
|
|
1808
|
+
assert tools[0]["tool_name"] == tools[1]["tool_name"] == "adder"
|
|
1809
|
+
assert tools[0]["tool_call_id"] == tools[1]["tool_call_id"] == "call_abc"
|
|
1810
|
+
assert tools[1]["output"] == "3"
|
|
1811
|
+
# ...and the node itself is still exactly one hook, not two.
|
|
1812
|
+
assert [r["hook_name"] for r in only(rows, "hook_triggered")] == ["plan", "adder"]
|
|
1813
|
+
|
|
1814
|
+
|
|
1815
|
+
def test_a_node_named_after_its_model_still_records_the_model(tmp_path, instrumented):
|
|
1816
|
+
"""Same collision one run type over: the model name, both token counts and
|
|
1817
|
+
the latency were dropped while the trace still looked populated."""
|
|
1818
|
+
model = fake_model(tool_calling_message())
|
|
1819
|
+
|
|
1820
|
+
def call(state):
|
|
1821
|
+
return {"messages": [model.invoke(state["messages"])], "vals": ["x"]}
|
|
1822
|
+
|
|
1823
|
+
# `GenericFakeChatModel`'s run name is its class name.
|
|
1824
|
+
app = build_simple([("GenericFakeChatModel", call)], name="model-collide")
|
|
1825
|
+
app.invoke(empty_state(), config={"configurable": {"thread_id": "model-collision"}})
|
|
1826
|
+
|
|
1827
|
+
rows = read_events(tmp_path)
|
|
1828
|
+
assert types_of(only(rows, "model_request", "model_response")) == [
|
|
1829
|
+
"model_request",
|
|
1830
|
+
"model_response",
|
|
1831
|
+
], "the chat model run was misfiled as a second visit to the node of the same name"
|
|
1832
|
+
response = only(rows, "model_response")[0]
|
|
1833
|
+
assert response["input_tokens"] == 11
|
|
1834
|
+
assert response["output_tokens"] == 5
|
|
1835
|
+
assert isinstance(response["duration_ms"], int)
|
|
1836
|
+
assert len(only(rows, "hook_triggered")) == 1
|
|
1837
|
+
|
|
1838
|
+
|
|
1839
|
+
def test_an_inner_runnable_sharing_the_node_name_is_not_a_second_visit(
|
|
1840
|
+
tmp_path, instrumented
|
|
1841
|
+
):
|
|
1842
|
+
"""`add_node("same", something.with_config(run_name="same"))` produced TWO
|
|
1843
|
+
`hook_triggered`/`hook_completed` pairs for one visit, doubling that node's
|
|
1844
|
+
count on `/hooks` and halving its apparent latency."""
|
|
1845
|
+
inner = RunnableLambda(lambda state: {"vals": ["same"]}).with_config(run_name="same")
|
|
1846
|
+
app = build_simple([("same", inner)], name="dup")
|
|
1847
|
+
app.invoke(empty_state(), config={"configurable": {"thread_id": "dup"}})
|
|
1848
|
+
|
|
1849
|
+
rows = read_events(tmp_path)
|
|
1850
|
+
assert types_of(only(rows, "hook_triggered", "hook_completed")) == [
|
|
1851
|
+
"hook_triggered",
|
|
1852
|
+
"hook_completed",
|
|
1853
|
+
]
|
|
1854
|
+
|
|
1855
|
+
|
|
1856
|
+
def test_the_node_exclusions_do_not_swallow_real_nodes(tmp_path, instrumented):
|
|
1857
|
+
"""The counterweight. Both new conditions are exclusions, and `_node_of`
|
|
1858
|
+
gates `hook_triggered` AND `_ensure_subgraph_agent` — over-tighten it and
|
|
1859
|
+
the timeline loses every node and every subgraph agent at once."""
|
|
1860
|
+
app = build_graph()
|
|
1861
|
+
app.invoke(empty_state(), config={"configurable": {"thread_id": "counterweight"}})
|
|
1862
|
+
|
|
1863
|
+
rows = read_events(tmp_path)
|
|
1864
|
+
names = [r["hook_name"] for r in only(rows, "hook_triggered")]
|
|
1865
|
+
# A plain function node, a ToolNode, a compiled subgraph as a node, two
|
|
1866
|
+
# Send-dispatched copies of one node: all still hooks.
|
|
1867
|
+
assert names.count("plan") == 1
|
|
1868
|
+
assert names.count("tools") == 1
|
|
1869
|
+
assert names.count("child") == 1
|
|
1870
|
+
assert names.count("worker") == 2
|
|
1871
|
+
assert names.count("sub_step") == 1
|
|
1872
|
+
# And the subgraph is still a nested agent, which only happens from inside
|
|
1873
|
+
# the node branch of `_on_start`.
|
|
1874
|
+
assert [r["agent_id"] for r in only(rows, "agent_start")] == [
|
|
1875
|
+
"root_graph",
|
|
1876
|
+
"root_graph/child",
|
|
1877
|
+
]
|
|
1878
|
+
|
|
1879
|
+
|
|
1880
|
+
# ---------------------------------------------------------------------------
|
|
1881
|
+
# A run that merely OVERLAPS a pause is not the approval
|
|
1882
|
+
# ---------------------------------------------------------------------------
|
|
1883
|
+
|
|
1884
|
+
|
|
1885
|
+
def test_an_unrelated_run_during_a_pause_is_not_read_as_the_approval(
|
|
1886
|
+
tmp_path, instrumented
|
|
1887
|
+
):
|
|
1888
|
+
"""A HITL turn sits paused on a human for minutes. Any other run carrying
|
|
1889
|
+
the same session id in that window — a second request on one conversation
|
|
1890
|
+
id, a background summariser, a different graph entirely — was read as the
|
|
1891
|
+
answer: it got no `agent_start` of its own, its work was folded into the
|
|
1892
|
+
paused span, and the adapter emitted `agent_resume` + `human_input` for a
|
|
1893
|
+
human who had answered nothing. Fabricating an approval is the worst wrong
|
|
1894
|
+
answer a human-approval product can give."""
|
|
1895
|
+
session = {"metadata": {adapter.SESSION_METADATA_KEY: "one-conversation"}}
|
|
1896
|
+
paused = build_graph(checkpointer=InMemorySaver())
|
|
1897
|
+
paused.invoke(
|
|
1898
|
+
empty_state(), config={"configurable": {"thread_id": "held"}, **session}
|
|
1899
|
+
)
|
|
1900
|
+
assert types_of(only(read_events(tmp_path), "agent_pause")) == ["agent_pause"]
|
|
1901
|
+
|
|
1902
|
+
unrelated = build_simple([("summarise", lambda s: {"vals": ["s"]})], name="other")
|
|
1903
|
+
unrelated.invoke(empty_state(), config=dict(session))
|
|
1904
|
+
|
|
1905
|
+
rows = read_events(tmp_path)
|
|
1906
|
+
assert not only(rows, "agent_resume"), "an unrelated run closed the human's pause"
|
|
1907
|
+
assert not only(rows, "human_input"), "an approval was recorded that never happened"
|
|
1908
|
+
# The unrelated run is its own agent, with its own span, not a relabelled
|
|
1909
|
+
# continuation of the paused one.
|
|
1910
|
+
assert "other" in [r["agent_id"] for r in only(rows, "agent_start")]
|
|
1911
|
+
assert "other" in [r["agent_id"] for r in only(rows, "agent_end")]
|
|
1912
|
+
|
|
1913
|
+
|
|
1914
|
+
def test_a_none_input_is_still_a_continuation_of_the_pause(tmp_path, instrumented):
|
|
1915
|
+
"""The counterweight for `_is_continuation`: `invoke(None, config)` is
|
|
1916
|
+
langgraph's other documented way to resume, and narrowing the test to
|
|
1917
|
+
`Command` alone would split that run in two."""
|
|
1918
|
+
app = build_graph(checkpointer=InMemorySaver())
|
|
1919
|
+
config = {"configurable": {"thread_id": "none-resume"}}
|
|
1920
|
+
app.invoke(empty_state(), config=config)
|
|
1921
|
+
app.invoke(None, config=config)
|
|
1922
|
+
|
|
1923
|
+
rows = read_events(tmp_path)
|
|
1924
|
+
roots = [r for r in only(rows, "agent_start") if not r.get("parent_id")]
|
|
1925
|
+
assert len(roots) == 1, "invoke(None) opened a second root span"
|
|
1926
|
+
# No answer was supplied, so the node interrupts again and the pause
|
|
1927
|
+
# reopens — but the FIRST one was closed, on the same span.
|
|
1928
|
+
assert types_of(only(rows, "agent_pause", "agent_resume"))[:3] == [
|
|
1929
|
+
"agent_pause",
|
|
1930
|
+
"agent_resume",
|
|
1931
|
+
"agent_pause",
|
|
1932
|
+
]
|
|
1933
|
+
|
|
1934
|
+
|
|
1935
|
+
# ---------------------------------------------------------------------------
|
|
1936
|
+
# The resume arrives in a DIFFERENT PROCESS
|
|
1937
|
+
# ---------------------------------------------------------------------------
|
|
1938
|
+
#
|
|
1939
|
+
# Every pause above is keyed on the `Interrupt` object the pausing process saw,
|
|
1940
|
+
# which assumes the process that paused is the process that resumes. Real HITL
|
|
1941
|
+
# is not shaped like that: one worker serves the request that interrupts, a
|
|
1942
|
+
# human answers minutes later, and whichever worker picks that request up
|
|
1943
|
+
# resumes against the shared checkpointer. `_STATE` is per process, so the
|
|
1944
|
+
# resuming worker had no `_Session` and no `open_pauses` — it emitted no
|
|
1945
|
+
# `agent_resume` and no `human_input` at all, and the `human_wait` /
|
|
1946
|
+
# `agent_pause` from the first worker stayed open FOREVER. Every cross-process
|
|
1947
|
+
# approval left its session reporting "still waiting on a human" after the human
|
|
1948
|
+
# had answered.
|
|
1949
|
+
#
|
|
1950
|
+
# `_STATE.reset()` between the two `.invoke()` calls is exactly a fresh process:
|
|
1951
|
+
# it is the same clear the adapter does at `install()`, and the events the first
|
|
1952
|
+
# "process" wrote are already on disk.
|
|
1953
|
+
|
|
1954
|
+
|
|
1955
|
+
def _forget_everything_this_process_knows():
|
|
1956
|
+
adapter._STATE.reset()
|
|
1957
|
+
|
|
1958
|
+
|
|
1959
|
+
def test_a_resume_from_another_process_still_closes_the_pause(tmp_path, instrumented):
|
|
1960
|
+
saver = InMemorySaver()
|
|
1961
|
+
config = {"configurable": {"thread_id": "xproc"}}
|
|
1962
|
+
|
|
1963
|
+
build_graph(checkpointer=saver).invoke(empty_state(), config=config)
|
|
1964
|
+
opened = only(read_events(tmp_path), "human_wait")
|
|
1965
|
+
assert len(opened) == 1
|
|
1966
|
+
|
|
1967
|
+
_forget_everything_this_process_knows()
|
|
1968
|
+
build_graph(checkpointer=saver).invoke(Command(resume="approved"), config=config)
|
|
1969
|
+
|
|
1970
|
+
rows = read_events(tmp_path)
|
|
1971
|
+
resumes = only(rows, "agent_resume")
|
|
1972
|
+
answers = only(rows, "human_input")
|
|
1973
|
+
assert len(resumes) == 1 and len(answers) == 1, (
|
|
1974
|
+
"the pause opened by the other process was never closed"
|
|
1975
|
+
)
|
|
1976
|
+
# Correlated on the id the FIRST process reported, with no shared state:
|
|
1977
|
+
# langgraph derives `Interrupt.id` from the interrupted task's checkpoint
|
|
1978
|
+
# namespace, which is byte-identical across the two invocations.
|
|
1979
|
+
assert resumes[0]["pause_id"] == opened[0]["input_id"]
|
|
1980
|
+
assert answers[0]["input_id"] == opened[0]["input_id"]
|
|
1981
|
+
assert answers[0]["response"] == "approved"
|
|
1982
|
+
|
|
1983
|
+
|
|
1984
|
+
def test_a_remote_resume_credits_the_interrupted_node_not_the_subgraph_host(
|
|
1985
|
+
tmp_path, instrumented
|
|
1986
|
+
):
|
|
1987
|
+
"""The subgraph host node re-runs too, at a shallower namespace, and it
|
|
1988
|
+
starts *before* the lifecycle event that unmasks it. Crediting the pause to
|
|
1989
|
+
it would emit an id that correlates with nothing and leave the real pause
|
|
1990
|
+
open — the exact failure being fixed, wearing a fix."""
|
|
1991
|
+
saver = InMemorySaver()
|
|
1992
|
+
config = {"configurable": {"thread_id": "xproc-sub"}}
|
|
1993
|
+
|
|
1994
|
+
def build():
|
|
1995
|
+
inner = StateGraph(State)
|
|
1996
|
+
inner.add_node("sub_pre", lambda state: {"vals": ["pre"]})
|
|
1997
|
+
inner.add_node("ask", lambda state: {"answer": str(interrupt({"prompt": "ok?"}))})
|
|
1998
|
+
inner.add_edge(START, "sub_pre")
|
|
1999
|
+
inner.add_edge("sub_pre", "ask")
|
|
2000
|
+
inner.add_edge("ask", END)
|
|
2001
|
+
outer = StateGraph(State)
|
|
2002
|
+
outer.add_node("before", lambda state: {"vals": ["b"]})
|
|
2003
|
+
outer.add_node("child", inner.compile(name="child_graph"))
|
|
2004
|
+
outer.add_node("after", lambda state: {"vals": ["a"]})
|
|
2005
|
+
outer.add_edge(START, "before")
|
|
2006
|
+
outer.add_edge("before", "child")
|
|
2007
|
+
outer.add_edge("child", "after")
|
|
2008
|
+
outer.add_edge("after", END)
|
|
2009
|
+
return outer.compile(name="root_graph", checkpointer=saver)
|
|
2010
|
+
|
|
2011
|
+
build().invoke(empty_state(), config=config)
|
|
2012
|
+
opened = only(read_events(tmp_path), "human_wait")
|
|
2013
|
+
assert len(opened) == 1
|
|
2014
|
+
|
|
2015
|
+
_forget_everything_this_process_knows()
|
|
2016
|
+
build().invoke(Command(resume="yes"), config=config)
|
|
2017
|
+
|
|
2018
|
+
rows = read_events(tmp_path)
|
|
2019
|
+
# Exactly one pair — not one per node that re-ran, and not one for the
|
|
2020
|
+
# subgraph host.
|
|
2021
|
+
assert len(only(rows, "agent_resume")) == 1
|
|
2022
|
+
assert len(only(rows, "human_input")) == 1
|
|
2023
|
+
assert only(rows, "agent_resume")[0]["pause_id"] == opened[0]["input_id"]
|
|
2024
|
+
|
|
2025
|
+
|
|
2026
|
+
def test_a_remote_resume_invents_no_pause_for_downstream_nodes(tmp_path, instrumented):
|
|
2027
|
+
"""Only the level's FIRST superstep re-runs interrupted tasks; every node
|
|
2028
|
+
after it is ordinary downstream work at the same namespace depth. Without
|
|
2029
|
+
that guard the resumed run manufactures one `agent_resume` + `human_input`
|
|
2030
|
+
per node it visits, each with an id that matches no pause — and the real
|
|
2031
|
+
pause still never closes."""
|
|
2032
|
+
saver = InMemorySaver()
|
|
2033
|
+
config = {"configurable": {"thread_id": "xproc-downstream"}}
|
|
2034
|
+
|
|
2035
|
+
def build():
|
|
2036
|
+
return build_simple(
|
|
2037
|
+
[
|
|
2038
|
+
("ask", lambda state: {"answer": str(interrupt({"prompt": "ok?"}))}),
|
|
2039
|
+
("after_one", lambda state: {"vals": ["a"]}),
|
|
2040
|
+
("after_two", lambda state: {"vals": ["b"]}),
|
|
2041
|
+
],
|
|
2042
|
+
name="downstream",
|
|
2043
|
+
checkpointer=saver,
|
|
2044
|
+
)
|
|
2045
|
+
|
|
2046
|
+
build().invoke(empty_state(), config=config)
|
|
2047
|
+
opened = only(read_events(tmp_path), "human_wait")
|
|
2048
|
+
assert len(opened) == 1
|
|
2049
|
+
|
|
2050
|
+
_forget_everything_this_process_knows()
|
|
2051
|
+
build().invoke(Command(resume="yes"), config=config)
|
|
2052
|
+
|
|
2053
|
+
rows = read_events(tmp_path)
|
|
2054
|
+
assert [r["hook_name"] for r in only(rows, "hook_triggered")][1:] == [
|
|
2055
|
+
"ask",
|
|
2056
|
+
"after_one",
|
|
2057
|
+
"after_two",
|
|
2058
|
+
]
|
|
2059
|
+
assert len(only(rows, "agent_resume")) == 1
|
|
2060
|
+
assert len(only(rows, "human_input")) == 1
|
|
2061
|
+
assert only(rows, "agent_resume")[0]["pause_id"] == opened[0]["input_id"]
|
|
2062
|
+
assert len(only(rows, "human_wait")) == 1
|
|
2063
|
+
assert not only(rows, "error")
|
|
2064
|
+
|
|
2065
|
+
|
|
2066
|
+
def test_a_fresh_turn_in_a_fresh_process_is_not_a_remote_resume(tmp_path, instrumented):
|
|
2067
|
+
"""The counterweight. `_RemoteResume` is armed off the root input alone, so
|
|
2068
|
+
a graph that simply runs — no interrupt anywhere, no `Command` — must never
|
|
2069
|
+
manufacture a resume for a human who was never asked."""
|
|
2070
|
+
build_graph().invoke(empty_state(), config={"configurable": {"thread_id": "plain"}})
|
|
2071
|
+
rows = read_events(tmp_path)
|
|
2072
|
+
assert not only(rows, "agent_resume")
|
|
2073
|
+
assert not only(rows, "human_input")
|
|
2074
|
+
|
|
2075
|
+
|
|
2076
|
+
def test_a_remote_rerun_with_no_answer_records_no_approval(tmp_path, instrumented):
|
|
2077
|
+
"""The sharp counterweight for the arming condition.
|
|
2078
|
+
|
|
2079
|
+
`invoke(None, config)` re-runs an interrupted thread WITHOUT answering it —
|
|
2080
|
+
the task interrupts again and no human said anything. Widening the arming
|
|
2081
|
+
test from "a `Command` carrying a resume value" to "anything that continues
|
|
2082
|
+
a thread" would record an approval, with an empty response, for a human who
|
|
2083
|
+
is still waiting. The pause must simply stay open."""
|
|
2084
|
+
saver = InMemorySaver()
|
|
2085
|
+
config = {"configurable": {"thread_id": "xproc-noanswer"}}
|
|
2086
|
+
build_graph(checkpointer=saver).invoke(empty_state(), config=config)
|
|
2087
|
+
|
|
2088
|
+
_forget_everything_this_process_knows()
|
|
2089
|
+
build_graph(checkpointer=saver).invoke(None, config=config)
|
|
2090
|
+
|
|
2091
|
+
rows = read_events(tmp_path)
|
|
2092
|
+
assert not only(rows, "agent_resume"), "an approval was recorded for a re-run"
|
|
2093
|
+
assert not only(rows, "human_input")
|
|
2094
|
+
# Still waiting, and said so twice — once per attempt.
|
|
2095
|
+
assert len(only(rows, "human_wait")) == 2
|
|
2096
|
+
|
|
2097
|
+
|
|
2098
|
+
def test_an_in_process_resume_emits_exactly_one_resume_pair(tmp_path, instrumented):
|
|
2099
|
+
"""The other counterweight: the remote path must stay inert whenever this
|
|
2100
|
+
process owns the pause, or every ordinary approval is recorded twice."""
|
|
2101
|
+
app = build_graph(checkpointer=InMemorySaver())
|
|
2102
|
+
config = {"configurable": {"thread_id": "in-process"}}
|
|
2103
|
+
app.invoke(empty_state(), config=config)
|
|
2104
|
+
app.invoke(Command(resume="yes"), config=config)
|
|
2105
|
+
|
|
2106
|
+
rows = read_events(tmp_path)
|
|
2107
|
+
assert len(only(rows, "agent_resume")) == 1
|
|
2108
|
+
assert len(only(rows, "human_input")) == 1
|
|
2109
|
+
assert only(rows, "agent_resume")[0].get("fw_resumed_elsewhere") is None
|
|
2110
|
+
|
|
2111
|
+
|
|
2112
|
+
def test_the_interrupt_id_is_still_derived_from_the_checkpoint_namespace():
|
|
2113
|
+
"""The whole cross-process fix rests on one langgraph invariant: an
|
|
2114
|
+
interrupt's id is `xxh3_128` of the interrupted task's checkpoint namespace,
|
|
2115
|
+
not a random value. If upstream makes it random, `_close_remote_pause` goes
|
|
2116
|
+
on emitting ids that correlate with nothing and every behavioural test above
|
|
2117
|
+
keeps passing, because they all compare our derived id against itself."""
|
|
2118
|
+
from langgraph.types import Interrupt
|
|
2119
|
+
|
|
2120
|
+
ns = "ask:0f1e2d3c-4b5a-6978-8796-a5b4c3d2e1f0"
|
|
2121
|
+
assert Interrupt.from_ns("value", ns).id == adapter._interrupt_id_of(ns)
|
|
2122
|
+
assert Interrupt.from_ns("a different value", ns).id == adapter._interrupt_id_of(ns)
|
|
2123
|
+
assert adapter._interrupt_id_of(ns) != adapter._interrupt_id_of(ns + "x")
|
|
2124
|
+
|
|
2125
|
+
|
|
2126
|
+
def test_a_subgraph_compiled_under_its_node_name_is_one_hook_not_two(
|
|
2127
|
+
tmp_path, instrumented
|
|
2128
|
+
):
|
|
2129
|
+
"""`sub.compile(name="child")` added as `add_node("child", sub)` is the
|
|
2130
|
+
natural way to name a subgraph, and it makes the subgraph's own Pregel run
|
|
2131
|
+
match the node too. That produced a duplicate `hook_triggered` for the node
|
|
2132
|
+
AND turned the Pregel run into a node, which then also opened and closed a
|
|
2133
|
+
nested agent on the same run — one visit rendering as four spans."""
|
|
2134
|
+
inner = StateGraph(State)
|
|
2135
|
+
inner.add_node("deep", lambda state: {"vals": ["deep"]})
|
|
2136
|
+
inner.add_edge(START, "deep")
|
|
2137
|
+
inner.add_edge("deep", END)
|
|
2138
|
+
|
|
2139
|
+
outer = StateGraph(State)
|
|
2140
|
+
outer.add_node("child", inner.compile(name="child"))
|
|
2141
|
+
outer.add_edge(START, "child")
|
|
2142
|
+
outer.add_edge("child", END)
|
|
2143
|
+
outer.compile(name="root_graph").invoke(
|
|
2144
|
+
empty_state(), config={"configurable": {"thread_id": "sub-name"}}
|
|
2145
|
+
)
|
|
2146
|
+
|
|
2147
|
+
rows = read_events(tmp_path)
|
|
2148
|
+
assert [r["hook_name"] for r in only(rows, "hook_triggered")] == ["child", "deep"]
|
|
2149
|
+
assert [r["hook_name"] for r in only(rows, "hook_completed")] == ["deep", "child"]
|
|
2150
|
+
assert [r["agent_id"] for r in only(rows, "agent_start")] == [
|
|
2151
|
+
"root_graph",
|
|
2152
|
+
"root_graph/child",
|
|
2153
|
+
]
|
|
2154
|
+
|
|
2155
|
+
|
|
2156
|
+
def test_a_leaf_run_is_never_the_nodes_own_run(tmp_path):
|
|
2157
|
+
"""The second exclusion, pinned directly.
|
|
2158
|
+
|
|
2159
|
+
langgraph 1.2.11 happens to tag the inner run `seq:step:N` as well, so the
|
|
2160
|
+
behavioural tests above would still pass with this condition removed — and
|
|
2161
|
+
that is exactly why it is here. It states the invariant that does not depend
|
|
2162
|
+
on a tag convention: whatever you hand `add_node`, the node's OWN run is the
|
|
2163
|
+
`chain` run langgraph builds around it, and a `tool` / `llm` / `chat_model` /
|
|
2164
|
+
`retriever` run carrying the node's name is the thing you passed, running
|
|
2165
|
+
underneath. Lose this and a tag rename silently deletes tool and model
|
|
2166
|
+
events again, which is a wrong answer with no symptom.
|
|
2167
|
+
"""
|
|
2168
|
+
import types as _types
|
|
2169
|
+
|
|
2170
|
+
meta = {"langgraph_node": "adder"}
|
|
2171
|
+
|
|
2172
|
+
def run(run_type, tags=()):
|
|
2173
|
+
return _types.SimpleNamespace(name="adder", run_type=run_type, tags=list(tags))
|
|
2174
|
+
|
|
2175
|
+
assert adapter._node_of(run("chain"), meta) == "adder"
|
|
2176
|
+
for leaf in ("tool", "llm", "chat_model", "retriever"):
|
|
2177
|
+
assert adapter._node_of(run(leaf), meta) is None, (
|
|
2178
|
+
f"a {leaf} run named after its node was claimed as the node itself"
|
|
2179
|
+
)
|
|
2180
|
+
|
|
2181
|
+
|
|
2182
|
+
def test_an_overlapping_run_cannot_strand_the_pause_it_did_not_answer(
|
|
2183
|
+
tmp_path, instrumented
|
|
2184
|
+
):
|
|
2185
|
+
"""The two fixes above meeting in the shape that produced both.
|
|
2186
|
+
|
|
2187
|
+
`_State.sessions` is keyed by session id, so a second root under the same id
|
|
2188
|
+
overwrites the paused run's entry and then pops it on the way out. The real
|
|
2189
|
+
approval, arriving afterwards, finds nothing — which is the same position a
|
|
2190
|
+
fresh worker is in, and is why the remote path is the backstop rather than a
|
|
2191
|
+
special case. The pause must still close, on the id the human was asked
|
|
2192
|
+
under, carrying what they actually said."""
|
|
2193
|
+
session = {"metadata": {adapter.SESSION_METADATA_KEY: "shared-conversation"}}
|
|
2194
|
+
app = build_graph(checkpointer=InMemorySaver())
|
|
2195
|
+
config = {"configurable": {"thread_id": "stranded"}, **session}
|
|
2196
|
+
|
|
2197
|
+
app.invoke(empty_state(), config=config)
|
|
2198
|
+
opened = only(read_events(tmp_path), "human_wait")
|
|
2199
|
+
assert len(opened) == 1
|
|
2200
|
+
|
|
2201
|
+
build_simple([("other", lambda state: {"vals": ["o"]})], name="other").invoke(
|
|
2202
|
+
empty_state(), config=dict(session)
|
|
2203
|
+
)
|
|
2204
|
+
app.invoke(Command(resume="approved"), config=config)
|
|
2205
|
+
|
|
2206
|
+
rows = read_events(tmp_path)
|
|
2207
|
+
assert {r["session_id"] for r in rows} == {"shared-conversation"}
|
|
2208
|
+
assert len(only(rows, "agent_resume")) == 1
|
|
2209
|
+
answers = only(rows, "human_input")
|
|
2210
|
+
assert len(answers) == 1
|
|
2211
|
+
assert answers[0]["input_id"] == opened[0]["input_id"]
|
|
2212
|
+
assert answers[0]["response"] == "approved"
|
|
2213
|
+
|
|
2214
|
+
|
|
2215
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
2216
|
+
# capture_limit — how much of a prompt or a tool output survives
|
|
2217
|
+
# ─────────────────────────────────────────────────────────────────────────────
|
|
2218
|
+
#
|
|
2219
|
+
# This adapter used to pin every captured value at 2048 characters, below the
|
|
2220
|
+
# core's 8192, on the grounds that "inputs/outputs are graph state". That is true
|
|
2221
|
+
# of a state blob and false of the two values people open this integration to
|
|
2222
|
+
# read. A real RAG prompt clears 2 KiB comfortably, so the tightening fired on
|
|
2223
|
+
# essentially every production run and showed the first paragraph of the prompt.
|
|
2224
|
+
|
|
2225
|
+
|
|
2226
|
+
def test_the_default_capture_limit_is_the_core_field_limit():
|
|
2227
|
+
"""Not 2048. The tightening below the core default is gone."""
|
|
2228
|
+
assert adapter._FIELD_LIMIT == _core.FIELD_LIMIT
|
|
2229
|
+
assert adapter._read_options({}).capture_limit == _core.FIELD_LIMIT
|
|
2230
|
+
|
|
2231
|
+
|
|
2232
|
+
def test_capture_limit_is_honoured(monkeypatch):
|
|
2233
|
+
monkeypatch.setattr(adapter._STATE, "options", adapter._read_options({"capture_limit": 32_768}))
|
|
2234
|
+
assert adapter._field_limit() == 32_768
|
|
2235
|
+
|
|
2236
|
+
|
|
2237
|
+
def test_capture_limit_accepts_a_string(monkeypatch):
|
|
2238
|
+
"""`instrument()` options routinely arrive from env plumbing as strings."""
|
|
2239
|
+
monkeypatch.setattr(adapter._STATE, "options", adapter._read_options({"capture_limit": "16384"}))
|
|
2240
|
+
assert adapter._field_limit() == 16_384
|
|
2241
|
+
|
|
2242
|
+
|
|
2243
|
+
@pytest.mark.parametrize("bad", [0, -1, "nope", 3.5 + 0j, object()])
|
|
2244
|
+
def test_an_unusable_capture_limit_falls_back_rather_than_raising(bad, monkeypatch, sink):
|
|
2245
|
+
"""`instrument()` with no name installs EVERY detected adapter with the same
|
|
2246
|
+
options, so a value meant for another framework must not break this one."""
|
|
2247
|
+
sink.allow = True
|
|
2248
|
+
monkeypatch.setattr(adapter._STATE, "options", adapter._read_options({"capture_limit": bad}))
|
|
2249
|
+
assert adapter._field_limit() == _core.FIELD_LIMIT
|
|
2250
|
+
|
|
2251
|
+
|
|
2252
|
+
def test_capture_limit_is_not_reported_as_an_unknown_option(sink):
|
|
2253
|
+
"""It must be in the known-options set, or every install logs about it."""
|
|
2254
|
+
adapter._read_options({"capture_limit": 4096})
|
|
2255
|
+
assert not [r for r in sink.records if "ignoring options" in r.getMessage()]
|
|
2256
|
+
|
|
2257
|
+
|
|
2258
|
+
def test_the_limit_is_read_per_call_not_bound_at_import(monkeypatch):
|
|
2259
|
+
"""The trap this adapter must not repeat.
|
|
2260
|
+
|
|
2261
|
+
`_core.FIELD_LIMIT` is exported in `__all__`, but reassigning it does
|
|
2262
|
+
nothing, because `def truncate(value, limit=FIELD_LIMIT)` binds the default
|
|
2263
|
+
at def time. A limit captured into a default argument here would ignore
|
|
2264
|
+
`capture_limit` in exactly the same way.
|
|
2265
|
+
"""
|
|
2266
|
+
monkeypatch.setattr(adapter._STATE, "options", adapter._read_options({"capture_limit": 1234}))
|
|
2267
|
+
assert adapter._field_limit() == 1234
|
|
2268
|
+
monkeypatch.setattr(adapter._STATE, "options", adapter._read_options({"capture_limit": 5678}))
|
|
2269
|
+
assert adapter._field_limit() == 5678
|
|
2270
|
+
|
|
2271
|
+
|
|
2272
|
+
# ---------------------------------------------------------------------------
|
|
2273
|
+
# `None` is not, by itself, evidence of a resume
|
|
2274
|
+
# ---------------------------------------------------------------------------
|
|
2275
|
+
|
|
2276
|
+
def test_resuming_a_graph_with_a_bare_none_is_still_read_as_a_continuation(
|
|
2277
|
+
tmp_path, instrumented
|
|
2278
|
+
):
|
|
2279
|
+
"""`graph.invoke(None, config)` is LangGraph's documented no-value resume.
|
|
2280
|
+
|
|
2281
|
+
Guards the fix below from over-correcting: requiring positive evidence that a
|
|
2282
|
+
run is a graph run must not stop recognising the real thing. This graph
|
|
2283
|
+
interrupts unconditionally, so `None` (which supplies no answer) interrupts
|
|
2284
|
+
it a second time — the assertion is therefore that it stayed ONE run, not
|
|
2285
|
+
that it completed.
|
|
2286
|
+
"""
|
|
2287
|
+
app = build_graph(checkpointer=InMemorySaver())
|
|
2288
|
+
config = {"configurable": {"thread_id": "hitl-none"}}
|
|
2289
|
+
first = app.invoke(empty_state(), config=config)
|
|
2290
|
+
assert "__interrupt__" in first
|
|
2291
|
+
app.invoke(None, config=config)
|
|
2292
|
+
|
|
2293
|
+
rows = read_events(tmp_path)
|
|
2294
|
+
# One run, one ROOT span: a continuation folds into the open root rather
|
|
2295
|
+
# than opening a second one. (A subgraph child span is expected and is not
|
|
2296
|
+
# a second root.)
|
|
2297
|
+
roots = [e for e in only(rows, "agent_start") if e["agent_id"] == "root_graph"]
|
|
2298
|
+
assert len(roots) == 1, "the resume opened a second root instead of continuing"
|
|
2299
|
+
assert roots[0]["session_id"] == "hitl-none"
|
|
2300
|
+
assert adapter._is_graph_run(
|
|
2301
|
+
type("R", (), {"metadata": {"thread_id": "t"}, "inputs": {"input": None}})()
|
|
2302
|
+
)
|
|
2303
|
+
|
|
2304
|
+
|
|
2305
|
+
def test_an_unrelated_runnable_invoked_with_none_is_not_the_humans_approval(
|
|
2306
|
+
tmp_path, instrumented
|
|
2307
|
+
):
|
|
2308
|
+
"""`{"input": None}` is not specific to LangGraph.
|
|
2309
|
+
|
|
2310
|
+
langchain-core wraps ANY non-mapping input to ANY root runnable under the
|
|
2311
|
+
`input` key, so `some_runnable.invoke(None)` produces byte-for-byte the shape
|
|
2312
|
+
a no-value resume does. Combined with the resume branch in `_start_root`,
|
|
2313
|
+
that made an unrelated run during a pause — a heartbeat, a summariser — get
|
|
2314
|
+
read as the human's answer: no `agent_start` of its own, its events folded
|
|
2315
|
+
into the paused root, and an `agent_resume` + `human_input(response=None)`
|
|
2316
|
+
emitted for a human who answered nothing.
|
|
2317
|
+
|
|
2318
|
+
The sibling guard test only ever invoked the unrelated runnable with a fresh
|
|
2319
|
+
STATE DICT, never with `None`, which is why this shape got through.
|
|
2320
|
+
"""
|
|
2321
|
+
app = build_graph(checkpointer=InMemorySaver())
|
|
2322
|
+
config = {"configurable": {"thread_id": "hitl-unrelated"}}
|
|
2323
|
+
first = app.invoke(empty_state(), config=config)
|
|
2324
|
+
assert "__interrupt__" in first
|
|
2325
|
+
|
|
2326
|
+
from langchain_core.runnables import RunnableLambda
|
|
2327
|
+
|
|
2328
|
+
RunnableLambda(lambda _: "beat").invoke(None)
|
|
2329
|
+
|
|
2330
|
+
rows = read_events(tmp_path)
|
|
2331
|
+
# The pause is still open: nothing answered it.
|
|
2332
|
+
assert only(rows, "human_input") == []
|
|
2333
|
+
assert only(rows, "agent_resume") == []
|
|
2334
|
+
# And the heartbeat got a root span of its own, in its own session, rather
|
|
2335
|
+
# than being swallowed into the paused graph run.
|
|
2336
|
+
beats = [e for e in only(rows, "agent_start") if e["agent_id"] == "RunnableLambda"]
|
|
2337
|
+
assert len(beats) == 1
|
|
2338
|
+
assert beats[0]["session_id"] != "hitl-unrelated"
|