@damiangilgonzalez/copypasta 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +227 -2
- package/catalog.yaml +140 -0
- package/cli/copypasta.js +4 -0
- package/cli/lib/catalog.js +131 -0
- package/cli/lib/commands.js +162 -0
- package/cli/lib/index.js +4 -0
- package/cli/lib/main.js +130 -0
- package/package.json +38 -4
- package/recipes/agents/agent-memory/RECIPE.md +75 -0
- package/recipes/agents/agent-memory/files/agentic/memory/__init__.py +25 -0
- package/recipes/agents/agent-memory/files/agentic/memory/base.py +101 -0
- package/recipes/agents/agent-memory/files/agentic/memory/factory.py +33 -0
- package/recipes/agents/agent-memory/files/agentic/memory/in_memory.py +59 -0
- package/recipes/agents/agent-memory/files/agentic/memory/models.py +66 -0
- package/recipes/agents/agent-memory/files/agentic/memory/settings.py +24 -0
- package/recipes/agents/agent-memory/files/agentic/memory/sql.py +177 -0
- package/recipes/agents/agent-memory/files/pytest.ini +5 -0
- package/recipes/agents/agent-memory/files/tests/__init__.py +0 -0
- package/recipes/agents/agent-memory/files/tests/memory_conformance.py +158 -0
- package/recipes/agents/agent-memory/files/tests/test_memory.py +74 -0
- package/recipes/agents/langchain-agent/RECIPE.md +101 -0
- package/recipes/agents/langchain-agent/files/agentic/__init__.py +10 -0
- package/recipes/agents/langchain-agent/files/agentic/agents/__init__.py +13 -0
- package/recipes/agents/langchain-agent/files/agentic/agents/agents.py +43 -0
- package/recipes/agents/langchain-agent/files/agentic/agents/base_agent.py +126 -0
- package/recipes/agents/langchain-agent/files/agentic/agents/models.py +46 -0
- package/recipes/agents/langchain-agent/files/agentic/context.py +21 -0
- package/recipes/agents/langchain-agent/files/agentic/llm/__init__.py +5 -0
- package/recipes/agents/langchain-agent/files/agentic/llm/models.py +45 -0
- package/recipes/agents/langchain-agent/files/agentic/llm/provider.py +78 -0
- package/recipes/agents/langchain-agent/files/agentic/llm/settings.py +62 -0
- package/recipes/agents/langchain-agent/files/agentic/middleware/__init__.py +3 -0
- package/recipes/agents/langchain-agent/files/agentic/middleware/language.py +28 -0
- package/recipes/agents/langchain-agent/files/agentic/prompts/__init__.py +3 -0
- package/recipes/agents/langchain-agent/files/agentic/prompts/assistant.en.yaml +15 -0
- package/recipes/agents/langchain-agent/files/agentic/prompts/assistant.es.yaml +15 -0
- package/recipes/agents/langchain-agent/files/agentic/prompts/loader.py +69 -0
- package/recipes/agents/langchain-agent/files/agentic/tools/__init__.py +25 -0
- package/recipes/agents/langchain-agent/files/agentic/tools/base_tool.py +62 -0
- package/recipes/agents/langchain-agent/files/agentic/tools/search_documents.py +42 -0
- package/recipes/agents/langchain-agent/files/pytest.ini +5 -0
- package/recipes/agents/langchain-agent/files/tests/__init__.py +0 -0
- package/recipes/agents/langchain-agent/files/tests/fakes.py +30 -0
- package/recipes/agents/langchain-agent/files/tests/test_agents.py +151 -0
- package/recipes/agents/langgraph-graph/RECIPE.md +63 -0
- package/recipes/agents/langgraph-graph/files/agentic/conversation.py +73 -0
- package/recipes/agents/langgraph-graph/files/agentic/graph/__init__.py +17 -0
- package/recipes/agents/langgraph-graph/files/agentic/graph/builder.py +37 -0
- package/recipes/agents/langgraph-graph/files/agentic/graph/edges.py +8 -0
- package/recipes/agents/langgraph-graph/files/agentic/graph/event_processor.py +48 -0
- package/recipes/agents/langgraph-graph/files/agentic/graph/events.py +30 -0
- package/recipes/agents/langgraph-graph/files/agentic/graph/graph.py +60 -0
- package/recipes/agents/langgraph-graph/files/agentic/graph/nodes.py +63 -0
- package/recipes/agents/langgraph-graph/files/agentic/graph/state.py +31 -0
- package/recipes/agents/langgraph-graph/files/agentic/registry.py +85 -0
- package/recipes/agents/langgraph-graph/files/tests/test_graph.py +145 -0
- package/recipes/backend/.gitkeep +0 -0
- package/recipes/backend/agent-chat-api/RECIPE.md +69 -0
- package/recipes/backend/agent-chat-api/files/api/agent/guards.py +35 -0
- package/recipes/backend/agent-chat-api/files/api/agent/router.py +86 -0
- package/recipes/backend/agent-chat-api/files/api/agent/sanitize.py +22 -0
- package/recipes/backend/agent-chat-api/files/api/agent/schemas.py +26 -0
- package/recipes/backend/agent-chat-api/files/api/agent/settings.py +14 -0
- package/recipes/backend/agent-chat-api/files/api/agent/tracing.py +38 -0
- package/recipes/backend/agent-chat-api/files/api/app.py +67 -0
- package/recipes/backend/agent-chat-api/files/api/auth.py +73 -0
- package/recipes/backend/agent-chat-api/files/pytest.ini +5 -0
- package/recipes/backend/agent-chat-api/files/tests/test_agent_api.py +237 -0
- package/recipes/backend/agent-feedback/RECIPE.md +63 -0
- package/recipes/backend/agent-feedback/files/api/feedback/__init__.py +0 -0
- package/recipes/backend/agent-feedback/files/api/feedback/router.py +72 -0
- package/recipes/backend/agent-feedback/files/api/feedback/schemas.py +44 -0
- package/recipes/backend/agent-feedback/files/feedback/__init__.py +25 -0
- package/recipes/backend/agent-feedback/files/feedback/base.py +53 -0
- package/recipes/backend/agent-feedback/files/feedback/factory.py +30 -0
- package/recipes/backend/agent-feedback/files/feedback/in_memory.py +29 -0
- package/recipes/backend/agent-feedback/files/feedback/models.py +48 -0
- package/recipes/backend/agent-feedback/files/feedback/settings.py +17 -0
- package/recipes/backend/agent-feedback/files/feedback/sql.py +102 -0
- package/recipes/backend/agent-feedback/files/pytest.ini +5 -0
- package/recipes/backend/agent-feedback/files/tests/feedback_conformance.py +89 -0
- package/recipes/backend/agent-feedback/files/tests/test_feedback.py +63 -0
- package/recipes/backend/agent-feedback/files/tests/test_feedback_api.py +95 -0
- package/recipes/backend/auth-verifier/RECIPE.md +63 -0
- package/recipes/backend/auth-verifier/files/api/auth.py +73 -0
- package/recipes/backend/auth-verifier/files/auth/__init__.py +23 -0
- package/recipes/backend/auth-verifier/files/auth/factory.py +28 -0
- package/recipes/backend/auth-verifier/files/auth/jwt_verifier.py +101 -0
- package/recipes/backend/auth-verifier/files/auth/models.py +29 -0
- package/recipes/backend/auth-verifier/files/auth/settings.py +45 -0
- package/recipes/backend/auth-verifier/files/auth/supabase.py +80 -0
- package/recipes/backend/auth-verifier/files/pytest.ini +5 -0
- package/recipes/backend/auth-verifier/files/tests/__init__.py +0 -0
- package/recipes/backend/auth-verifier/files/tests/test_auth.py +229 -0
- package/recipes/backend/conversation-history/RECIPE.md +49 -0
- package/recipes/backend/conversation-history/files/api/conversations/__init__.py +0 -0
- package/recipes/backend/conversation-history/files/api/conversations/router.py +55 -0
- package/recipes/backend/conversation-history/files/api/conversations/schemas.py +57 -0
- package/recipes/backend/conversation-history/files/pytest.ini +5 -0
- package/recipes/backend/conversation-history/files/tests/test_conversations_api.py +127 -0
- package/recipes/backend/fastapi-base/RECIPE.md +57 -0
- package/recipes/backend/fastapi-base/files/api/app.py +37 -0
- package/recipes/backend/fastapi-base/files/api/base.py +54 -0
- package/recipes/backend/fastapi-base/files/api/errors.py +31 -0
- package/recipes/backend/fastapi-base/files/api/health.py +19 -0
- package/recipes/backend/fastapi-base/files/api/logging_setup.py +26 -0
- package/recipes/backend/fastapi-base/files/api/request_id.py +62 -0
- package/recipes/backend/fastapi-base/files/api/settings.py +32 -0
- package/recipes/backend/fastapi-base/files/pytest.ini +5 -0
- package/recipes/backend/fastapi-base/files/tests/__init__.py +0 -0
- package/recipes/backend/fastapi-base/files/tests/test_api_base.py +136 -0
- package/recipes/backend/observability/RECIPE.md +68 -0
- package/recipes/backend/observability/files/observability/__init__.py +34 -0
- package/recipes/backend/observability/files/observability/factory.py +73 -0
- package/recipes/backend/observability/files/observability/langfuse_tracer.py +29 -0
- package/recipes/backend/observability/files/observability/langsmith_tracer.py +23 -0
- package/recipes/backend/observability/files/observability/models.py +90 -0
- package/recipes/backend/observability/files/observability/opik_tracer.py +30 -0
- package/recipes/backend/observability/files/observability/settings.py +58 -0
- package/recipes/backend/observability/files/observability/tracer.py +142 -0
- package/recipes/backend/observability/files/observability/usage.py +61 -0
- package/recipes/backend/observability/files/pytest.ini +5 -0
- package/recipes/backend/observability/files/tests/__init__.py +0 -0
- package/recipes/backend/observability/files/tests/test_observability.py +219 -0
- package/recipes/backend/project-settings/RECIPE.md +40 -0
- package/recipes/backend/project-settings/files/.env.example +7 -0
- package/recipes/backend/project-settings/files/core/__init__.py +0 -0
- package/recipes/backend/project-settings/files/core/config.py +41 -0
- package/recipes/backend/project-settings/files/pytest.ini +3 -0
- package/recipes/backend/project-settings/files/tests/__init__.py +0 -0
- package/recipes/backend/project-settings/files/tests/test_settings.py +44 -0
- package/recipes/backend/rag-retriever/RECIPE.md +79 -0
- package/recipes/backend/rag-retriever/files/pytest.ini +5 -0
- package/recipes/backend/rag-retriever/files/rag/__init__.py +53 -0
- package/recipes/backend/rag-retriever/files/rag/base.py +106 -0
- package/recipes/backend/rag-retriever/files/rag/chunking.py +84 -0
- package/recipes/backend/rag-retriever/files/rag/document_search.py +60 -0
- package/recipes/backend/rag-retriever/files/rag/embeddings.py +91 -0
- package/recipes/backend/rag-retriever/files/rag/factory.py +82 -0
- package/recipes/backend/rag-retriever/files/rag/in_memory.py +52 -0
- package/recipes/backend/rag-retriever/files/rag/models.py +107 -0
- package/recipes/backend/rag-retriever/files/rag/pgvector.py +138 -0
- package/recipes/backend/rag-retriever/files/rag/qdrant.py +96 -0
- package/recipes/backend/rag-retriever/files/rag/settings.py +48 -0
- package/recipes/backend/rag-retriever/files/tests/__init__.py +0 -0
- package/recipes/backend/rag-retriever/files/tests/rag_conformance.py +95 -0
- package/recipes/backend/rag-retriever/files/tests/test_rag.py +270 -0
- package/recipes/backend/rate-limit-quota/RECIPE.md +77 -0
- package/recipes/backend/rate-limit-quota/files/api/agent/guards.py +62 -0
- package/recipes/backend/rate-limit-quota/files/api/limits/__init__.py +0 -0
- package/recipes/backend/rate-limit-quota/files/api/limits/router.py +38 -0
- package/recipes/backend/rate-limit-quota/files/limits/__init__.py +43 -0
- package/recipes/backend/rate-limit-quota/files/limits/base.py +161 -0
- package/recipes/backend/rate-limit-quota/files/limits/factory.py +69 -0
- package/recipes/backend/rate-limit-quota/files/limits/in_memory.py +36 -0
- package/recipes/backend/rate-limit-quota/files/limits/models.py +108 -0
- package/recipes/backend/rate-limit-quota/files/limits/settings.py +32 -0
- package/recipes/backend/rate-limit-quota/files/limits/sql.py +94 -0
- package/recipes/backend/rate-limit-quota/files/pytest.ini +5 -0
- package/recipes/backend/rate-limit-quota/files/tests/limits_conformance.py +146 -0
- package/recipes/backend/rate-limit-quota/files/tests/test_limits.py +167 -0
- package/recipes/backend/rate-limit-quota/files/tests/test_limits_api.py +100 -0
- package/recipes/backend/voice-providers/RECIPE.md +73 -0
- package/recipes/backend/voice-providers/files/api/__init__.py +0 -0
- package/recipes/backend/voice-providers/files/api/voice/__init__.py +0 -0
- package/recipes/backend/voice-providers/files/api/voice/router.py +113 -0
- package/recipes/backend/voice-providers/files/pytest.ini +5 -0
- package/recipes/backend/voice-providers/files/tests/__init__.py +0 -0
- package/recipes/backend/voice-providers/files/tests/fake_openai.py +44 -0
- package/recipes/backend/voice-providers/files/tests/test_voice.py +164 -0
- package/recipes/backend/voice-providers/files/tests/test_voice_api.py +106 -0
- package/recipes/backend/voice-providers/files/tests/voice_conformance.py +80 -0
- package/recipes/backend/voice-providers/files/voice/__init__.py +29 -0
- package/recipes/backend/voice-providers/files/voice/base.py +116 -0
- package/recipes/backend/voice-providers/files/voice/factory.py +48 -0
- package/recipes/backend/voice-providers/files/voice/fake.py +51 -0
- package/recipes/backend/voice-providers/files/voice/models.py +85 -0
- package/recipes/backend/voice-providers/files/voice/openai_provider.py +116 -0
- package/recipes/backend/voice-providers/files/voice/settings.py +48 -0
- package/recipes/frontend/.gitkeep +0 -0
- package/recipes/frontend/playwright-e2e/RECIPE.md +48 -0
- package/recipes/frontend/playwright-e2e/files/.github/workflows/frontend.yml +35 -0
- package/recipes/frontend/playwright-e2e/files/e2e/helpers.ts +25 -0
- package/recipes/frontend/playwright-e2e/files/e2e/home.spec.ts +10 -0
- package/recipes/frontend/playwright-e2e/files/e2e/login.spec.ts +26 -0
- package/recipes/frontend/playwright-e2e/files/e2e/navigation.spec.ts +27 -0
- package/recipes/frontend/playwright-e2e/files/playwright.config.ts +35 -0
- package/recipes/frontend/react-agent-chat/RECIPE.md +69 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/chat-api.ts +27 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/chat.css +18 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/chat.test.tsx +139 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/components/ChatView.tsx +22 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/components/Composer.tsx +40 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/components/Markdown.tsx +20 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/components/MessageList.tsx +42 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/events.ts +42 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/fake-chat-api.ts +44 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/index.ts +6 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/sse-parser.test.ts +97 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/sse-parser.ts +85 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/use-chat.ts +136 -0
- package/recipes/frontend/react-app-router/RECIPE.md +58 -0
- package/recipes/frontend/react-app-router/files/src/App.test.tsx +54 -0
- package/recipes/frontend/react-app-router/files/src/App.tsx +14 -0
- package/recipes/frontend/react-app-router/files/src/router/ErrorScreen.tsx +21 -0
- package/recipes/frontend/react-app-router/files/src/router/Layout.tsx +28 -0
- package/recipes/frontend/react-app-router/files/src/router/NotFoundScreen.tsx +12 -0
- package/recipes/frontend/react-app-router/files/src/router/index.ts +1 -0
- package/recipes/frontend/react-app-router/files/src/router/router.css +29 -0
- package/recipes/frontend/react-app-router/files/src/router/router.test.tsx +65 -0
- package/recipes/frontend/react-app-router/files/src/router/routes.tsx +59 -0
- package/recipes/frontend/react-app-router/files/src/screens/SettingsScreen.tsx +10 -0
- package/recipes/frontend/react-supabase-auth/RECIPE.md +77 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/LoginScreen.tsx +64 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/SignOutButton.tsx +12 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/auth.css +33 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/auth.test.ts +149 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/auth.ts +59 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/fake-auth.ts +87 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/fake-gotrue.ts +85 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/index.ts +7 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/make-auth.ts +20 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/session-context.tsx +35 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/session-store.ts +63 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/session.test.tsx +56 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/settings.ts +30 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/supabase-auth.ts +85 -0
- package/recipes/frontend/vite-react-base/RECIPE.md +69 -0
- package/recipes/frontend/vite-react-base/files/.env +6 -0
- package/recipes/frontend/vite-react-base/files/.env.fake +2 -0
- package/recipes/frontend/vite-react-base/files/index.html +12 -0
- package/recipes/frontend/vite-react-base/files/package.json +30 -0
- package/recipes/frontend/vite-react-base/files/src/App.test.tsx +49 -0
- package/recipes/frontend/vite-react-base/files/src/App.tsx +13 -0
- package/recipes/frontend/vite-react-base/files/src/api/api.ts +37 -0
- package/recipes/frontend/vite-react-base/files/src/api/fake/fake-api.ts +29 -0
- package/recipes/frontend/vite-react-base/files/src/api/http-api.ts +16 -0
- package/recipes/frontend/vite-react-base/files/src/api/http-client.test.ts +84 -0
- package/recipes/frontend/vite-react-base/files/src/api/http-client.ts +94 -0
- package/recipes/frontend/vite-react-base/files/src/api/make-api.ts +30 -0
- package/recipes/frontend/vite-react-base/files/src/api/types.ts +7 -0
- package/recipes/frontend/vite-react-base/files/src/app-context.tsx +24 -0
- package/recipes/frontend/vite-react-base/files/src/auth/token-provider.ts +27 -0
- package/recipes/frontend/vite-react-base/files/src/config.ts +18 -0
- package/recipes/frontend/vite-react-base/files/src/hooks/use-load.ts +46 -0
- package/recipes/frontend/vite-react-base/files/src/main.tsx +27 -0
- package/recipes/frontend/vite-react-base/files/src/screens/HomeScreen.tsx +28 -0
- package/recipes/frontend/vite-react-base/files/src/styles/global.css +25 -0
- package/recipes/frontend/vite-react-base/files/src/styles/tokens.css +33 -0
- package/recipes/frontend/vite-react-base/files/src/styles/tokens.test.ts +45 -0
- package/recipes/frontend/vite-react-base/files/src/test-setup.ts +8 -0
- package/recipes/frontend/vite-react-base/files/src/vite-env.d.ts +11 -0
- package/recipes/frontend/vite-react-base/files/tsconfig.json +18 -0
- package/recipes/frontend/vite-react-base/files/vite.config.ts +19 -0
- package/unpacked-files.json +3 -0
|
@@ -0,0 +1,68 @@
|
|
|
1
|
+
# observability
|
|
2
|
+
|
|
3
|
+
## Cuándo usarla
|
|
4
|
+
|
|
5
|
+
Para **ver qué hace cada turno del agente**: la traza completa (nodos, llamadas al LLM, tools) en Langfuse, Opik o LangSmith elegido en `.env`, y, con o sin proveedor, los tokens, el coste y la duración de cada turno en una línea de log y en un `TurnSummary` que puedes guardar. Sin configuración no traza nada y la app arranca igual.
|
|
6
|
+
|
|
7
|
+
No depende de otras recetas. **Usa `project-settings`** (bloque `ObservabilitySettings`) y **encaja con `agent-chat-api`** (`create_app(tracer=...)`), que ya llama a `begin_turn` y `finish` en cada turno.
|
|
8
|
+
|
|
9
|
+
## Qué incluye
|
|
10
|
+
|
|
11
|
+
```
|
|
12
|
+
observability/
|
|
13
|
+
models.py TracingProvider · RunInfo (thread, tenant, user, request_id, tags) · TokenUsage · ModelPrice · Prices · TurnSummary
|
|
14
|
+
usage.py UsageCollector (callback: tokens de cada llamada al chat model) · usage_from_messages (lo mismo sobre mensajes guardados)
|
|
15
|
+
tracer.py Tracer (ABC: _callbacks, _metadata, _trace_id, _flush) · NullTracer · TurnTrace (config del run + finish() → TurnSummary)
|
|
16
|
+
langfuse_tracer.py LangfuseTracer: un cliente por proceso, un CallbackHandler por run, session = thread_id, user = user_id
|
|
17
|
+
opik_tracer.py OpikTracer: OpikTracer del SDK por run con thread_id (vista Threads), tags y metadata
|
|
18
|
+
langsmith_tracer.py LangSmithTracer: LangChainTracer explícito con Client propio (nada por variables de entorno)
|
|
19
|
+
settings.py bloque ObservabilitySettings: TRACING_PROVIDER (none por defecto), TRACING_TAGS, LANGFUSE_*, OPIK_*, LANGSMITH_*
|
|
20
|
+
factory.py TracingFactory.create(settings, prices=): nunca lanza; proveedor que falla → NullTracer con warning
|
|
21
|
+
tests/test_observability.py 12 tests sin red: tokens y caché, coste con descuento y None sin precio, collector en un chat model,
|
|
22
|
+
metadata/tags al handler, fail-open (begin, flush, timeout), settings, factory con los tres SDK offline
|
|
23
|
+
pytest.ini
|
|
24
|
+
```
|
|
25
|
+
|
|
26
|
+
## Decisiones
|
|
27
|
+
|
|
28
|
+
- **El core no sabe quién observa.** `agentic/` solo recibe un `config` (`callbacks`, `metadata`, `tags`, `run_name`); quien lo construye es `Tracer.begin(...)` en la capa de la API. Los nodos nunca importan el SDK.
|
|
29
|
+
- **Un `Tracer` por proceso, un `TurnTrace` por turno.** `begin` construye el handler del proveedor para ese run con el `thread_id` del servidor (tenant como namespace, como `MemoryKey.thread_id`), el tenant, el usuario y el `X-Request-ID`: la traza y la línea de log se cruzan por el mismo id, algo que los proyectos de origen no llegaban a hacer.
|
|
30
|
+
- **Apagado por defecto y fail-open.** `TRACING_PROVIDER=none` no construye nada. Si el SDK falta o la configuración es mala, la factory deja `NullTracer` y un warning; si el handler falla al construirse, el turno corre sin traza; si el flush falla o tarda, se corta con timeout (`TRACING_FLUSH_TIMEOUT_SECONDS`). En el log solo va la clase del error: el texto puede llevar la clave.
|
|
31
|
+
- **Tokens y coste sin depender del proveedor.** `UsageCollector` lee `usage_metadata` de cada respuesta del chat model, con `input_token_details.cache_read`. El coste es `(entrada − caché)·p_in + caché·p_caché + salida·p_out` por millón. Los precios los pone el proyecto (`Prices`); un modelo sin precio da `cost_usd=None`, nunca 0: un coste desconocido no se reporta como gratis.
|
|
32
|
+
- **`TurnSummary` es el dato, la línea de log es la vista.** Lo que un proyecto guarda en su tabla de gasto diario o en filas de traza sale de aquí: `summary.model_dump()` a tu tabla de gasto, a `rate-limit-quota` (`QuotaUnit.TOKENS`) o a la página de admin.
|
|
33
|
+
- **LangSmith explícito, no por variables de entorno.** `LANGSMITH_TRACING=true` en el entorno traza cualquier LangChain del proceso, también en tests. Aquí es un `LangChainTracer` con `Client` propio, solo en los runs que pasan por `begin`.
|
|
34
|
+
- **Sin spans manuales ni filtrado del ruido de LangGraph.** Filtrar los nodos que no aportan o trazar a mano un clasificador o una búsqueda es una subclase de `Tracer` (`_callbacks` devolviendo tu handler) y no parte de esta receta.
|
|
35
|
+
- **Sin feedback ni prompts en el proveedor.** Los pulgares no se mandan como score, y los prompts viven en YAML (`langchain-agent`). Si quieres el score, `tracer.trace_id` del `TurnSummary` es lo que hay que guardar junto al `message_id`.
|
|
36
|
+
|
|
37
|
+
## Cómo adaptarla
|
|
38
|
+
|
|
39
|
+
1. Copia `files/` en la raíz del backend. Sustituye `{{app_name}}` en `observability/settings.py` (nombre de proyecto en Opik y LangSmith).
|
|
40
|
+
2. **Settings:** `class Settings(ObservabilitySettings, LLMSettings, BaseAppSettings)`. En el `.env`, uno de:
|
|
41
|
+
|
|
42
|
+
```
|
|
43
|
+
TRACING_PROVIDER=langfuse LANGFUSE_PUBLIC_KEY=pk-lf-... LANGFUSE_SECRET_KEY=sk-lf-... LANGFUSE_HOST=https://cloud.langfuse.com
|
|
44
|
+
TRACING_PROVIDER=opik OPIK_URL_OVERRIDE=http://localhost:5173/api (o OPIK_API_KEY + OPIK_WORKSPACE para Cloud)
|
|
45
|
+
TRACING_PROVIDER=langsmith LANGSMITH_API_KEY=lsv2_... LANGSMITH_PROJECT={{app_name}}
|
|
46
|
+
TRACING_TAGS=prod,eu
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
e instala solo el SDK que uses (`langfuse`, `opik` o `langsmith`).
|
|
50
|
+
3. **Precios** (opcional): `Prices(by_model={"gpt-4.1-mini": ModelPrice(input=0.4, output=1.6, cached_input=0.1)})` en tu `main.py` o desde un YAML del proyecto.
|
|
51
|
+
4. **`main.py`** con `agent-chat-api`:
|
|
52
|
+
|
|
53
|
+
```python
|
|
54
|
+
tracer = TracingFactory.create(settings, prices=prices)
|
|
55
|
+
app = create_app(settings=settings, ..., tracer=tracer) # cada turno: begin_turn -> config -> finish; flush al apagar
|
|
56
|
+
```
|
|
57
|
+
|
|
58
|
+
Sin `agent-chat-api`: `trace = tracer.begin(RunInfo(thread_id=..., tenant_id=..., user_id=..., request_id=current_request_id()))`, pasa `trace.config` a `answer_question(..., config=...)` o a `graph.ainvoke(..., config=...)`, y en un `finally` `summary = trace.finish(error)`. Al apagar, `await tracer.flush()`.
|
|
59
|
+
5. **Guardar el gasto:** en `traced_turn` (o donde llames a `finish`) toma el `TurnSummary` y escríbelo donde toque; `usage_from_messages(new_messages)` da lo mismo a partir de la memoria si prefieres calcularlo al persistir.
|
|
60
|
+
6. **Proveedor nuevo o spans propios:** subclase de `Tracer` con `_callbacks(run)`, `_metadata(run)`, `_trace_id(run)` y `_flush()`, y un caso en `TracingFactory`.
|
|
61
|
+
7. Para comprobarla: `pytest` (12 tests; los de cada SDK se saltan si no está instalado). Con `agent-chat-api` y sus requeridas: 98 tests.
|
|
62
|
+
|
|
63
|
+
**Dependencias:** `langchain-core`, `pydantic-settings`; `langfuse>=3` (probado con 5.2), `opik` (2.2) o `langsmith` (0.15) según el proveedor.
|
|
64
|
+
|
|
65
|
+
## Origen
|
|
66
|
+
|
|
67
|
+
- Probado en producción: tracer construido una vez con import perezoso del SDK, apagado si falta la configuración y `[]` con warning si falla; `thread_id` del servidor en `metadata`; `Tracer.begin(...) -> TurnTrace` con `callbacks`, `metadata`, `finish` y `flush` con timeout en hilo; `NullTracer`; cada llamada al SDK en `try/except` y solo la clase del error al log; coste `None` si el modelo no tiene precio; cliente `Langfuse` por proceso y handler por turno; tokens desde `usage_metadata` con `cache_read` y coste por millón; registro de gasto que nunca propaga errores.
|
|
68
|
+
- Nuevo en esta receta: `request_id` en la traza (en producción estaba en el log pero no se cruzaba con la traza) y `LangChainTracer` explícito en lugar de LangSmith por variables de entorno, que trazaba también en tests.
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
"""Traces, tokens and cost of every agent turn, with the provider chosen in settings.
|
|
2
|
+
|
|
3
|
+
tracer = TracingFactory.create(settings, prices=prices) # startup, once
|
|
4
|
+
trace = tracer.begin(RunInfo(thread_id=..., tenant_id=..., user_id=..., request_id=...))
|
|
5
|
+
try:
|
|
6
|
+
state = await ainvoke_graph(question, context=context, config=trace.config)
|
|
7
|
+
finally:
|
|
8
|
+
summary = trace.finish() # tokens, cost, ms -> log + TurnSummary
|
|
9
|
+
await tracer.flush() # shutdown
|
|
10
|
+
|
|
11
|
+
The core (`agentic/`) never imports this package: it receives a `config` and that is all.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from .factory import TracingFactory
|
|
15
|
+
from .models import ModelPrice, Prices, RunInfo, TokenUsage, TracingProvider, TurnSummary
|
|
16
|
+
from .settings import ObservabilitySettings
|
|
17
|
+
from .tracer import NullTracer, Tracer, TurnTrace
|
|
18
|
+
from .usage import UsageCollector, usage_from_messages
|
|
19
|
+
|
|
20
|
+
__all__ = [
|
|
21
|
+
"ModelPrice",
|
|
22
|
+
"NullTracer",
|
|
23
|
+
"ObservabilitySettings",
|
|
24
|
+
"Prices",
|
|
25
|
+
"RunInfo",
|
|
26
|
+
"TokenUsage",
|
|
27
|
+
"Tracer",
|
|
28
|
+
"TracingFactory",
|
|
29
|
+
"TracingProvider",
|
|
30
|
+
"TurnSummary",
|
|
31
|
+
"TurnTrace",
|
|
32
|
+
"UsageCollector",
|
|
33
|
+
"usage_from_messages",
|
|
34
|
+
]
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""The one place that builds the `Tracer` the settings name. Never raises: a provider that cannot
|
|
2
|
+
be built (missing package, bad config) logs a warning and becomes `NullTracer`, so the app starts
|
|
3
|
+
and the turns run without traces."""
|
|
4
|
+
|
|
5
|
+
import logging
|
|
6
|
+
|
|
7
|
+
from .models import Prices, TracingProvider
|
|
8
|
+
from .settings import ObservabilitySettings
|
|
9
|
+
from .tracer import NullTracer, Tracer
|
|
10
|
+
|
|
11
|
+
logger = logging.getLogger(__name__)
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class TracingFactory:
|
|
15
|
+
@staticmethod
|
|
16
|
+
def create(settings: ObservabilitySettings, *, prices: Prices | None = None) -> Tracer:
|
|
17
|
+
"""Call it once at startup (FastAPI `lifespan`): `app.state.tracer = TracingFactory.create(settings)`.
|
|
18
|
+
At shutdown: `await app.state.tracer.flush()`."""
|
|
19
|
+
common = {"prices": prices, "flush_timeout": settings.tracing_flush_timeout_seconds}
|
|
20
|
+
if settings.tracing_provider is TracingProvider.NONE:
|
|
21
|
+
return NullTracer(**common)
|
|
22
|
+
try:
|
|
23
|
+
match settings.tracing_provider:
|
|
24
|
+
case TracingProvider.LANGFUSE:
|
|
25
|
+
return TracingFactory._langfuse(settings, **common)
|
|
26
|
+
case TracingProvider.OPIK:
|
|
27
|
+
return TracingFactory._opik(settings, **common)
|
|
28
|
+
case TracingProvider.LANGSMITH:
|
|
29
|
+
return TracingFactory._langsmith(settings, **common)
|
|
30
|
+
except Exception as exc:
|
|
31
|
+
logger.warning("tracing provider %s unavailable (%s): running without traces", settings.tracing_provider, type(exc).__name__)
|
|
32
|
+
return NullTracer(**common)
|
|
33
|
+
raise ValueError(f"Unknown tracing provider: {settings.tracing_provider!r}")
|
|
34
|
+
|
|
35
|
+
@staticmethod
|
|
36
|
+
def _langfuse(settings: ObservabilitySettings, **common) -> Tracer:
|
|
37
|
+
from langfuse import Langfuse
|
|
38
|
+
|
|
39
|
+
from .langfuse_tracer import LangfuseTracer
|
|
40
|
+
|
|
41
|
+
assert settings.langfuse_public_key and settings.langfuse_secret_key
|
|
42
|
+
public_key = settings.langfuse_public_key.get_secret_value()
|
|
43
|
+
client = Langfuse(
|
|
44
|
+
public_key=public_key, secret_key=settings.langfuse_secret_key.get_secret_value(), host=settings.langfuse_host
|
|
45
|
+
)
|
|
46
|
+
return LangfuseTracer(client, public_key=public_key, **common)
|
|
47
|
+
|
|
48
|
+
@staticmethod
|
|
49
|
+
def _opik(settings: ObservabilitySettings, **common) -> Tracer:
|
|
50
|
+
import opik
|
|
51
|
+
|
|
52
|
+
from .opik_tracer import OpikTracer
|
|
53
|
+
|
|
54
|
+
kwargs: dict = {"automatic_approvals": True, "install_mcp": False}
|
|
55
|
+
if settings.opik_api_key:
|
|
56
|
+
kwargs["api_key"] = settings.opik_api_key.get_secret_value()
|
|
57
|
+
kwargs["workspace"] = settings.opik_workspace
|
|
58
|
+
else:
|
|
59
|
+
kwargs["use_local"] = True
|
|
60
|
+
if settings.opik_url_override:
|
|
61
|
+
kwargs["url_override"] = settings.opik_url_override
|
|
62
|
+
opik.configure(**kwargs)
|
|
63
|
+
return OpikTracer(project_name=settings.opik_project_name, **common)
|
|
64
|
+
|
|
65
|
+
@staticmethod
|
|
66
|
+
def _langsmith(settings: ObservabilitySettings, **common) -> Tracer:
|
|
67
|
+
from langsmith import Client
|
|
68
|
+
|
|
69
|
+
from .langsmith_tracer import LangSmithTracer
|
|
70
|
+
|
|
71
|
+
assert settings.langsmith_api_key
|
|
72
|
+
client = Client(api_key=settings.langsmith_api_key.get_secret_value(), api_url=settings.langsmith_endpoint)
|
|
73
|
+
return LangSmithTracer(client, project_name=settings.langsmith_project, **common)
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""Langfuse (SDK v3+): one `Langfuse` client per process, one `CallbackHandler` per run.
|
|
2
|
+
Session = thread_id, user = user_id, both through the metadata keys the handler understands."""
|
|
3
|
+
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from .models import RunInfo
|
|
7
|
+
from .tracer import Tracer
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class LangfuseTracer(Tracer):
|
|
11
|
+
def __init__(self, client: Any, *, public_key: str, **kwargs: Any) -> None:
|
|
12
|
+
super().__init__(**kwargs)
|
|
13
|
+
self._client = client
|
|
14
|
+
self._public_key = public_key
|
|
15
|
+
|
|
16
|
+
def _callbacks(self, run: RunInfo) -> list[Any]:
|
|
17
|
+
from langfuse.langchain import CallbackHandler # lazy: install `langfuse` only if you use it
|
|
18
|
+
|
|
19
|
+
return [CallbackHandler(public_key=self._public_key)]
|
|
20
|
+
|
|
21
|
+
def _metadata(self, run: RunInfo) -> dict[str, Any]:
|
|
22
|
+
return {
|
|
23
|
+
"langfuse_session_id": run.thread_id,
|
|
24
|
+
"langfuse_user_id": run.user_id,
|
|
25
|
+
"langfuse_tags": list(run.tags),
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
def _flush(self) -> None:
|
|
29
|
+
self._client.flush()
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
"""LangSmith through an explicit `LangChainTracer` (one `Client` per process), instead of the
|
|
2
|
+
`LANGSMITH_TRACING` env vars: nothing traces by accident, and tests never reach the cloud.
|
|
3
|
+
`metadata` and `tags` of the run config land on the trace."""
|
|
4
|
+
|
|
5
|
+
from typing import Any
|
|
6
|
+
|
|
7
|
+
from .models import RunInfo
|
|
8
|
+
from .tracer import Tracer
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class LangSmithTracer(Tracer):
|
|
12
|
+
def __init__(self, client: Any, *, project_name: str, **kwargs: Any) -> None:
|
|
13
|
+
super().__init__(**kwargs)
|
|
14
|
+
self._client = client
|
|
15
|
+
self.project_name = project_name
|
|
16
|
+
|
|
17
|
+
def _callbacks(self, run: RunInfo) -> list[Any]:
|
|
18
|
+
from langchain_core.tracers.langchain import LangChainTracer
|
|
19
|
+
|
|
20
|
+
return [LangChainTracer(project_name=self.project_name, client=self._client)]
|
|
21
|
+
|
|
22
|
+
def _flush(self) -> None:
|
|
23
|
+
self._client.flush()
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
"""Data of the observability layer. Pydantic, frozen where it is a record."""
|
|
2
|
+
|
|
3
|
+
from enum import StrEnum
|
|
4
|
+
|
|
5
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
class TracingProvider(StrEnum):
|
|
9
|
+
NONE = "none"
|
|
10
|
+
LANGFUSE = "langfuse"
|
|
11
|
+
OPIK = "opik"
|
|
12
|
+
LANGSMITH = "langsmith"
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class RunInfo(BaseModel):
|
|
16
|
+
"""Who and what a run is about. Goes to the trace as metadata/tags; the LLM never sees it."""
|
|
17
|
+
|
|
18
|
+
model_config = ConfigDict(frozen=True)
|
|
19
|
+
|
|
20
|
+
thread_id: str = Field(description="Server-side conversation id (with the tenant as namespace).")
|
|
21
|
+
tenant_id: str
|
|
22
|
+
user_id: str
|
|
23
|
+
request_id: str | None = Field(default=None, description="X-Request-ID of the HTTP request, to cross logs and trace.")
|
|
24
|
+
name: str = "turn"
|
|
25
|
+
tags: list[str] = Field(default_factory=list)
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class TokenUsage(BaseModel):
|
|
29
|
+
"""One LLM call, from `AIMessage.usage_metadata`."""
|
|
30
|
+
|
|
31
|
+
model_config = ConfigDict(frozen=True)
|
|
32
|
+
|
|
33
|
+
model: str = "unknown"
|
|
34
|
+
input_tokens: int = 0
|
|
35
|
+
output_tokens: int = 0
|
|
36
|
+
cached_input_tokens: int = Field(default=0, description="Part of input_tokens served from the prompt cache.")
|
|
37
|
+
|
|
38
|
+
@property
|
|
39
|
+
def total_tokens(self) -> int:
|
|
40
|
+
return self.input_tokens + self.output_tokens
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
class ModelPrice(BaseModel):
|
|
44
|
+
"""USD per million tokens."""
|
|
45
|
+
|
|
46
|
+
model_config = ConfigDict(frozen=True)
|
|
47
|
+
|
|
48
|
+
input: float = Field(ge=0)
|
|
49
|
+
output: float = Field(ge=0)
|
|
50
|
+
cached_input: float | None = Field(default=None, ge=0, description="None -> same as input.")
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class Prices(BaseModel):
|
|
54
|
+
"""`{model_name: ModelPrice}`. Keep it in the project (settings or a YAML): prices change."""
|
|
55
|
+
|
|
56
|
+
model_config = ConfigDict(frozen=True)
|
|
57
|
+
|
|
58
|
+
by_model: dict[str, ModelPrice] = Field(default_factory=dict)
|
|
59
|
+
|
|
60
|
+
def cost_usd(self, usage: TokenUsage) -> float | None:
|
|
61
|
+
"""None when the model has no price: an unknown cost must not be reported as 0."""
|
|
62
|
+
price = self.by_model.get(usage.model)
|
|
63
|
+
if price is None:
|
|
64
|
+
return None
|
|
65
|
+
cached_price = price.input if price.cached_input is None else price.cached_input
|
|
66
|
+
fresh = usage.input_tokens - usage.cached_input_tokens
|
|
67
|
+
return (fresh * price.input + usage.cached_input_tokens * cached_price + usage.output_tokens * price.output) / 1e6
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
class TurnSummary(BaseModel):
|
|
71
|
+
"""What one turn cost: for the log line, the spend table or the admin page."""
|
|
72
|
+
|
|
73
|
+
model_config = ConfigDict(frozen=True)
|
|
74
|
+
|
|
75
|
+
thread_id: str
|
|
76
|
+
tenant_id: str
|
|
77
|
+
user_id: str
|
|
78
|
+
request_id: str | None = None
|
|
79
|
+
llm_calls: int
|
|
80
|
+
input_tokens: int
|
|
81
|
+
output_tokens: int
|
|
82
|
+
cached_input_tokens: int
|
|
83
|
+
cost_usd: float | None = Field(default=None, description="None if any call has no price.")
|
|
84
|
+
duration_ms: int
|
|
85
|
+
error: str | None = Field(default=None, description="Class name of the failure, never its text.")
|
|
86
|
+
trace_id: str | None = Field(default=None, description="Provider trace id when it is known before flushing.")
|
|
87
|
+
|
|
88
|
+
@property
|
|
89
|
+
def total_tokens(self) -> int:
|
|
90
|
+
return self.input_tokens + self.output_tokens
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""Opik (self-hosted or Cloud): `opik.configure(...)` once at build time (factory), one
|
|
2
|
+
`OpikTracer` per run with the thread id so the Threads view groups the conversation."""
|
|
3
|
+
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from .models import RunInfo
|
|
7
|
+
from .tracer import Tracer
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class OpikTracer(Tracer):
|
|
11
|
+
def __init__(self, *, project_name: str, **kwargs: Any) -> None:
|
|
12
|
+
super().__init__(**kwargs)
|
|
13
|
+
self.project_name = project_name
|
|
14
|
+
|
|
15
|
+
def _callbacks(self, run: RunInfo) -> list[Any]:
|
|
16
|
+
from opik.integrations.langchain import OpikTracer as OpikCallback # lazy
|
|
17
|
+
|
|
18
|
+
return [
|
|
19
|
+
OpikCallback(
|
|
20
|
+
project_name=self.project_name,
|
|
21
|
+
thread_id=run.thread_id,
|
|
22
|
+
tags=list(run.tags) or None,
|
|
23
|
+
metadata={"tenant_id": run.tenant_id, "user_id": run.user_id, "request_id": run.request_id},
|
|
24
|
+
)
|
|
25
|
+
]
|
|
26
|
+
|
|
27
|
+
def _flush(self) -> None:
|
|
28
|
+
import opik
|
|
29
|
+
|
|
30
|
+
opik.flush_tracker()
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""Observability settings BLOCK, meant to be mixed into the project settings (`project-settings`):
|
|
2
|
+
|
|
3
|
+
class Settings(ObservabilitySettings, LLMSettings, BaseAppSettings): ...
|
|
4
|
+
|
|
5
|
+
Env vars (nothing traces unless TRACING_PROVIDER names a provider):
|
|
6
|
+
TRACING_PROVIDER=none|langfuse|opik|langsmith default none
|
|
7
|
+
TRACING_TAGS=prod,eu tags on every trace (comma-separated)
|
|
8
|
+
TRACING_FLUSH_TIMEOUT_SECONDS=2
|
|
9
|
+
LANGFUSE_PUBLIC_KEY LANGFUSE_SECRET_KEY LANGFUSE_HOST=https://cloud.langfuse.com
|
|
10
|
+
OPIK_URL_OVERRIDE (self-hosted, e.g. http://localhost:5173/api) OPIK_API_KEY + OPIK_WORKSPACE (Cloud) OPIK_PROJECT_NAME
|
|
11
|
+
LANGSMITH_API_KEY LANGSMITH_ENDPOINT=https://api.smith.langchain.com LANGSMITH_PROJECT
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from typing import Annotated
|
|
15
|
+
|
|
16
|
+
from pydantic import Field, SecretStr, field_validator, model_validator
|
|
17
|
+
from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict
|
|
18
|
+
|
|
19
|
+
from .models import TracingProvider
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class ObservabilitySettings(BaseSettings):
|
|
23
|
+
model_config = SettingsConfigDict(extra="ignore")
|
|
24
|
+
|
|
25
|
+
tracing_provider: TracingProvider = TracingProvider.NONE
|
|
26
|
+
tracing_tags: Annotated[list[str], NoDecode] = Field(default_factory=list)
|
|
27
|
+
tracing_flush_timeout_seconds: float = Field(default=2.0, gt=0)
|
|
28
|
+
|
|
29
|
+
langfuse_public_key: SecretStr | None = None
|
|
30
|
+
langfuse_secret_key: SecretStr | None = None
|
|
31
|
+
langfuse_host: str = "https://cloud.langfuse.com"
|
|
32
|
+
|
|
33
|
+
opik_url_override: str | None = None
|
|
34
|
+
opik_api_key: SecretStr | None = None
|
|
35
|
+
opik_workspace: str | None = None
|
|
36
|
+
opik_project_name: str = "{{app_name}}"
|
|
37
|
+
|
|
38
|
+
langsmith_api_key: SecretStr | None = None
|
|
39
|
+
langsmith_endpoint: str = "https://api.smith.langchain.com"
|
|
40
|
+
langsmith_project: str = "{{app_name}}"
|
|
41
|
+
|
|
42
|
+
@field_validator("tracing_tags", mode="before")
|
|
43
|
+
@classmethod
|
|
44
|
+
def _split_commas(cls, value):
|
|
45
|
+
if isinstance(value, str):
|
|
46
|
+
return [tag.strip() for tag in value.split(",") if tag.strip()]
|
|
47
|
+
return value
|
|
48
|
+
|
|
49
|
+
@model_validator(mode="after")
|
|
50
|
+
def _provider_has_its_keys(self) -> "ObservabilitySettings":
|
|
51
|
+
match self.tracing_provider:
|
|
52
|
+
case TracingProvider.LANGFUSE if not (self.langfuse_public_key and self.langfuse_secret_key):
|
|
53
|
+
raise ValueError("TRACING_PROVIDER=langfuse needs LANGFUSE_PUBLIC_KEY and LANGFUSE_SECRET_KEY")
|
|
54
|
+
case TracingProvider.OPIK if not (self.opik_url_override or self.opik_api_key):
|
|
55
|
+
raise ValueError("TRACING_PROVIDER=opik needs OPIK_URL_OVERRIDE (self-hosted) or OPIK_API_KEY (Cloud)")
|
|
56
|
+
case TracingProvider.LANGSMITH if not self.langsmith_api_key:
|
|
57
|
+
raise ValueError("TRACING_PROVIDER=langsmith needs LANGSMITH_API_KEY")
|
|
58
|
+
return self
|
|
@@ -0,0 +1,142 @@
|
|
|
1
|
+
"""`Tracer`: one per process, built by `TracingFactory`. `begin(...)` gives a `TurnTrace` per run:
|
|
2
|
+
its `config` goes to `ainvoke` / `astream_events`, and `finish()` returns the `TurnSummary`
|
|
3
|
+
(tokens, cost, duration) and logs one line.
|
|
4
|
+
|
|
5
|
+
Fail-open everywhere: a provider that cannot build a handler, or that raises on flush, costs a
|
|
6
|
+
warning with the error's class name (never its text: it may hold a key) and nothing else.
|
|
7
|
+
The core (`agentic/`) never imports this module: it only receives a `config`.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
import asyncio
|
|
11
|
+
import logging
|
|
12
|
+
import time
|
|
13
|
+
from abc import ABC, abstractmethod
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
from langchain_core.runnables import RunnableConfig
|
|
17
|
+
|
|
18
|
+
from .models import Prices, RunInfo, TurnSummary
|
|
19
|
+
from .usage import UsageCollector
|
|
20
|
+
|
|
21
|
+
logger = logging.getLogger(__name__)
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class Tracer(ABC):
|
|
25
|
+
"""Subclasses implement `_callbacks(run)` (the provider's handler) and optionally `_metadata(run)`,
|
|
26
|
+
`_trace_id(run)` and `_flush()`."""
|
|
27
|
+
|
|
28
|
+
def __init__(self, *, prices: Prices | None = None, flush_timeout: float = 2.0) -> None:
|
|
29
|
+
self.prices = prices or Prices()
|
|
30
|
+
self.flush_timeout = flush_timeout
|
|
31
|
+
|
|
32
|
+
def begin(self, run: RunInfo) -> "TurnTrace":
|
|
33
|
+
callbacks: list[Any] = []
|
|
34
|
+
metadata: dict[str, Any] = {}
|
|
35
|
+
trace_id: str | None = None
|
|
36
|
+
try:
|
|
37
|
+
callbacks = list(self._callbacks(run))
|
|
38
|
+
metadata = dict(self._metadata(run))
|
|
39
|
+
trace_id = self._trace_id(run)
|
|
40
|
+
except Exception as exc:
|
|
41
|
+
logger.warning("tracing disabled for this run (%s)", type(exc).__name__)
|
|
42
|
+
callbacks, metadata, trace_id = [], {}, None
|
|
43
|
+
return TurnTrace(run, callbacks=callbacks, metadata=metadata, trace_id=trace_id, prices=self.prices)
|
|
44
|
+
|
|
45
|
+
def begin_turn(
|
|
46
|
+
self, *, thread_id: str, tenant_id: str, user_id: str, request_id: str | None = None, tags: list[str] = ()
|
|
47
|
+
) -> "TurnTrace":
|
|
48
|
+
"""`begin` with keyword arguments: what an API layer that does not import this package calls."""
|
|
49
|
+
return self.begin(
|
|
50
|
+
RunInfo(thread_id=thread_id, tenant_id=tenant_id, user_id=user_id, request_id=request_id, tags=list(tags))
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
async def flush(self) -> None:
|
|
54
|
+
"""Push what is pending, bounded by `flush_timeout`. Call it at shutdown (lifespan)."""
|
|
55
|
+
try:
|
|
56
|
+
await asyncio.wait_for(asyncio.to_thread(self._flush), timeout=self.flush_timeout)
|
|
57
|
+
except TimeoutError:
|
|
58
|
+
logger.warning("tracing flush timed out after %.1fs", self.flush_timeout)
|
|
59
|
+
except Exception as exc:
|
|
60
|
+
logger.warning("tracing flush failed (%s)", type(exc).__name__)
|
|
61
|
+
|
|
62
|
+
@abstractmethod
|
|
63
|
+
def _callbacks(self, run: RunInfo) -> list[Any]: ...
|
|
64
|
+
|
|
65
|
+
def _metadata(self, run: RunInfo) -> dict[str, Any]:
|
|
66
|
+
return {}
|
|
67
|
+
|
|
68
|
+
def _trace_id(self, run: RunInfo) -> str | None:
|
|
69
|
+
return None
|
|
70
|
+
|
|
71
|
+
def _flush(self) -> None:
|
|
72
|
+
return None
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
class NullTracer(Tracer):
|
|
76
|
+
"""No provider: the run still gets the usage collector and the base metadata."""
|
|
77
|
+
|
|
78
|
+
def _callbacks(self, run: RunInfo) -> list[Any]:
|
|
79
|
+
return []
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class TurnTrace:
|
|
83
|
+
"""One run. `config` carries the provider handler, the usage collector and the metadata."""
|
|
84
|
+
|
|
85
|
+
def __init__(
|
|
86
|
+
self,
|
|
87
|
+
run: RunInfo,
|
|
88
|
+
*,
|
|
89
|
+
callbacks: list[Any],
|
|
90
|
+
metadata: dict[str, Any],
|
|
91
|
+
trace_id: str | None,
|
|
92
|
+
prices: Prices,
|
|
93
|
+
) -> None:
|
|
94
|
+
self.run = run
|
|
95
|
+
self.trace_id = trace_id
|
|
96
|
+
self.usage = UsageCollector()
|
|
97
|
+
self._prices = prices
|
|
98
|
+
self._started = time.monotonic()
|
|
99
|
+
base_metadata = {
|
|
100
|
+
"thread_id": run.thread_id,
|
|
101
|
+
"tenant_id": run.tenant_id,
|
|
102
|
+
"user_id": run.user_id,
|
|
103
|
+
**({"request_id": run.request_id} if run.request_id else {}),
|
|
104
|
+
}
|
|
105
|
+
self.config: RunnableConfig = {
|
|
106
|
+
"callbacks": [*callbacks, self.usage],
|
|
107
|
+
"metadata": {**base_metadata, **metadata},
|
|
108
|
+
"tags": list(run.tags),
|
|
109
|
+
"run_name": run.name,
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
def finish(self, error: BaseException | None = None) -> TurnSummary:
|
|
113
|
+
"""Summarise and log one line. Call it in a `finally` after the run."""
|
|
114
|
+
costs = [self._prices.cost_usd(u) for u in self.usage.usages]
|
|
115
|
+
summary = TurnSummary(
|
|
116
|
+
thread_id=self.run.thread_id,
|
|
117
|
+
tenant_id=self.run.tenant_id,
|
|
118
|
+
user_id=self.run.user_id,
|
|
119
|
+
request_id=self.run.request_id,
|
|
120
|
+
llm_calls=len(self.usage.usages),
|
|
121
|
+
input_tokens=self.usage.input_tokens,
|
|
122
|
+
output_tokens=self.usage.output_tokens,
|
|
123
|
+
cached_input_tokens=self.usage.cached_input_tokens,
|
|
124
|
+
cost_usd=None if any(c is None for c in costs) else round(sum(costs), 6),
|
|
125
|
+
duration_ms=int((time.monotonic() - self._started) * 1000),
|
|
126
|
+
error=type(error).__name__ if error is not None else None,
|
|
127
|
+
trace_id=self.trace_id,
|
|
128
|
+
)
|
|
129
|
+
logger.info(
|
|
130
|
+
"turn thread=%s tenant=%s llm_calls=%d tokens=%d/%d cached=%d cost_usd=%s ms=%d%s%s",
|
|
131
|
+
summary.thread_id,
|
|
132
|
+
summary.tenant_id,
|
|
133
|
+
summary.llm_calls,
|
|
134
|
+
summary.input_tokens,
|
|
135
|
+
summary.output_tokens,
|
|
136
|
+
summary.cached_input_tokens,
|
|
137
|
+
"?" if summary.cost_usd is None else f"{summary.cost_usd:.6f}",
|
|
138
|
+
summary.duration_ms,
|
|
139
|
+
f" error={summary.error}" if summary.error else "",
|
|
140
|
+
f" trace={summary.trace_id}" if summary.trace_id else "",
|
|
141
|
+
)
|
|
142
|
+
return summary
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
"""Tokens per LLM call, provider-independent: a callback that reads `usage_metadata` from every
|
|
2
|
+
chat model result (`UsageCollector`), or the same read over persisted messages (`usage_from_messages`).
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from collections.abc import Iterable
|
|
6
|
+
from typing import Any
|
|
7
|
+
from uuid import UUID
|
|
8
|
+
|
|
9
|
+
from langchain_core.callbacks import AsyncCallbackHandler
|
|
10
|
+
from langchain_core.messages import AIMessage, BaseMessage
|
|
11
|
+
from langchain_core.outputs import LLMResult
|
|
12
|
+
|
|
13
|
+
from .models import TokenUsage
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def usage_of(message: AIMessage) -> TokenUsage | None:
|
|
17
|
+
meta = message.usage_metadata
|
|
18
|
+
if not meta:
|
|
19
|
+
return None
|
|
20
|
+
details = meta.get("input_token_details") or {}
|
|
21
|
+
return TokenUsage(
|
|
22
|
+
model=str((message.response_metadata or {}).get("model_name") or "unknown"),
|
|
23
|
+
input_tokens=int(meta.get("input_tokens", 0)),
|
|
24
|
+
output_tokens=int(meta.get("output_tokens", 0)),
|
|
25
|
+
cached_input_tokens=int(details.get("cache_read", 0) or 0),
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def usage_from_messages(messages: Iterable[BaseMessage]) -> list[TokenUsage]:
|
|
30
|
+
"""One entry per AI message that carries usage (tool calls included)."""
|
|
31
|
+
return [usage for m in messages if isinstance(m, AIMessage) and (usage := usage_of(m)) is not None]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class UsageCollector(AsyncCallbackHandler):
|
|
35
|
+
"""Collects `TokenUsage` of every chat model call of a run. Never raises into the run."""
|
|
36
|
+
|
|
37
|
+
raise_error = False
|
|
38
|
+
run_inline = True
|
|
39
|
+
|
|
40
|
+
def __init__(self) -> None:
|
|
41
|
+
super().__init__()
|
|
42
|
+
self.usages: list[TokenUsage] = []
|
|
43
|
+
|
|
44
|
+
async def on_llm_end(self, response: LLMResult, *, run_id: UUID, parent_run_id: UUID | None = None, **kwargs: Any) -> None:
|
|
45
|
+
for generations in response.generations:
|
|
46
|
+
for generation in generations:
|
|
47
|
+
message = getattr(generation, "message", None)
|
|
48
|
+
if isinstance(message, AIMessage) and (usage := usage_of(message)) is not None:
|
|
49
|
+
self.usages.append(usage)
|
|
50
|
+
|
|
51
|
+
@property
|
|
52
|
+
def input_tokens(self) -> int:
|
|
53
|
+
return sum(u.input_tokens for u in self.usages)
|
|
54
|
+
|
|
55
|
+
@property
|
|
56
|
+
def output_tokens(self) -> int:
|
|
57
|
+
return sum(u.output_tokens for u in self.usages)
|
|
58
|
+
|
|
59
|
+
@property
|
|
60
|
+
def cached_input_tokens(self) -> int:
|
|
61
|
+
return sum(u.cached_input_tokens for u in self.usages)
|
|
File without changes
|