@damiangilgonzalez/copypasta 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +227 -2
- package/catalog.yaml +140 -0
- package/cli/copypasta.js +4 -0
- package/cli/lib/catalog.js +131 -0
- package/cli/lib/commands.js +162 -0
- package/cli/lib/index.js +4 -0
- package/cli/lib/main.js +130 -0
- package/package.json +38 -4
- package/recipes/agents/agent-memory/RECIPE.md +75 -0
- package/recipes/agents/agent-memory/files/agentic/memory/__init__.py +25 -0
- package/recipes/agents/agent-memory/files/agentic/memory/base.py +101 -0
- package/recipes/agents/agent-memory/files/agentic/memory/factory.py +33 -0
- package/recipes/agents/agent-memory/files/agentic/memory/in_memory.py +59 -0
- package/recipes/agents/agent-memory/files/agentic/memory/models.py +66 -0
- package/recipes/agents/agent-memory/files/agentic/memory/settings.py +24 -0
- package/recipes/agents/agent-memory/files/agentic/memory/sql.py +177 -0
- package/recipes/agents/agent-memory/files/pytest.ini +5 -0
- package/recipes/agents/agent-memory/files/tests/__init__.py +0 -0
- package/recipes/agents/agent-memory/files/tests/memory_conformance.py +158 -0
- package/recipes/agents/agent-memory/files/tests/test_memory.py +74 -0
- package/recipes/agents/langchain-agent/RECIPE.md +101 -0
- package/recipes/agents/langchain-agent/files/agentic/__init__.py +10 -0
- package/recipes/agents/langchain-agent/files/agentic/agents/__init__.py +13 -0
- package/recipes/agents/langchain-agent/files/agentic/agents/agents.py +43 -0
- package/recipes/agents/langchain-agent/files/agentic/agents/base_agent.py +126 -0
- package/recipes/agents/langchain-agent/files/agentic/agents/models.py +46 -0
- package/recipes/agents/langchain-agent/files/agentic/context.py +21 -0
- package/recipes/agents/langchain-agent/files/agentic/llm/__init__.py +5 -0
- package/recipes/agents/langchain-agent/files/agentic/llm/models.py +45 -0
- package/recipes/agents/langchain-agent/files/agentic/llm/provider.py +78 -0
- package/recipes/agents/langchain-agent/files/agentic/llm/settings.py +62 -0
- package/recipes/agents/langchain-agent/files/agentic/middleware/__init__.py +3 -0
- package/recipes/agents/langchain-agent/files/agentic/middleware/language.py +28 -0
- package/recipes/agents/langchain-agent/files/agentic/prompts/__init__.py +3 -0
- package/recipes/agents/langchain-agent/files/agentic/prompts/assistant.en.yaml +15 -0
- package/recipes/agents/langchain-agent/files/agentic/prompts/assistant.es.yaml +15 -0
- package/recipes/agents/langchain-agent/files/agentic/prompts/loader.py +69 -0
- package/recipes/agents/langchain-agent/files/agentic/tools/__init__.py +25 -0
- package/recipes/agents/langchain-agent/files/agentic/tools/base_tool.py +62 -0
- package/recipes/agents/langchain-agent/files/agentic/tools/search_documents.py +42 -0
- package/recipes/agents/langchain-agent/files/pytest.ini +5 -0
- package/recipes/agents/langchain-agent/files/tests/__init__.py +0 -0
- package/recipes/agents/langchain-agent/files/tests/fakes.py +30 -0
- package/recipes/agents/langchain-agent/files/tests/test_agents.py +151 -0
- package/recipes/agents/langgraph-graph/RECIPE.md +63 -0
- package/recipes/agents/langgraph-graph/files/agentic/conversation.py +73 -0
- package/recipes/agents/langgraph-graph/files/agentic/graph/__init__.py +17 -0
- package/recipes/agents/langgraph-graph/files/agentic/graph/builder.py +37 -0
- package/recipes/agents/langgraph-graph/files/agentic/graph/edges.py +8 -0
- package/recipes/agents/langgraph-graph/files/agentic/graph/event_processor.py +48 -0
- package/recipes/agents/langgraph-graph/files/agentic/graph/events.py +30 -0
- package/recipes/agents/langgraph-graph/files/agentic/graph/graph.py +60 -0
- package/recipes/agents/langgraph-graph/files/agentic/graph/nodes.py +63 -0
- package/recipes/agents/langgraph-graph/files/agentic/graph/state.py +31 -0
- package/recipes/agents/langgraph-graph/files/agentic/registry.py +85 -0
- package/recipes/agents/langgraph-graph/files/tests/test_graph.py +145 -0
- package/recipes/backend/.gitkeep +0 -0
- package/recipes/backend/agent-chat-api/RECIPE.md +69 -0
- package/recipes/backend/agent-chat-api/files/api/agent/guards.py +35 -0
- package/recipes/backend/agent-chat-api/files/api/agent/router.py +86 -0
- package/recipes/backend/agent-chat-api/files/api/agent/sanitize.py +22 -0
- package/recipes/backend/agent-chat-api/files/api/agent/schemas.py +26 -0
- package/recipes/backend/agent-chat-api/files/api/agent/settings.py +14 -0
- package/recipes/backend/agent-chat-api/files/api/agent/tracing.py +38 -0
- package/recipes/backend/agent-chat-api/files/api/app.py +67 -0
- package/recipes/backend/agent-chat-api/files/api/auth.py +73 -0
- package/recipes/backend/agent-chat-api/files/pytest.ini +5 -0
- package/recipes/backend/agent-chat-api/files/tests/test_agent_api.py +237 -0
- package/recipes/backend/agent-feedback/RECIPE.md +63 -0
- package/recipes/backend/agent-feedback/files/api/feedback/__init__.py +0 -0
- package/recipes/backend/agent-feedback/files/api/feedback/router.py +72 -0
- package/recipes/backend/agent-feedback/files/api/feedback/schemas.py +44 -0
- package/recipes/backend/agent-feedback/files/feedback/__init__.py +25 -0
- package/recipes/backend/agent-feedback/files/feedback/base.py +53 -0
- package/recipes/backend/agent-feedback/files/feedback/factory.py +30 -0
- package/recipes/backend/agent-feedback/files/feedback/in_memory.py +29 -0
- package/recipes/backend/agent-feedback/files/feedback/models.py +48 -0
- package/recipes/backend/agent-feedback/files/feedback/settings.py +17 -0
- package/recipes/backend/agent-feedback/files/feedback/sql.py +102 -0
- package/recipes/backend/agent-feedback/files/pytest.ini +5 -0
- package/recipes/backend/agent-feedback/files/tests/feedback_conformance.py +89 -0
- package/recipes/backend/agent-feedback/files/tests/test_feedback.py +63 -0
- package/recipes/backend/agent-feedback/files/tests/test_feedback_api.py +95 -0
- package/recipes/backend/auth-verifier/RECIPE.md +63 -0
- package/recipes/backend/auth-verifier/files/api/auth.py +73 -0
- package/recipes/backend/auth-verifier/files/auth/__init__.py +23 -0
- package/recipes/backend/auth-verifier/files/auth/factory.py +28 -0
- package/recipes/backend/auth-verifier/files/auth/jwt_verifier.py +101 -0
- package/recipes/backend/auth-verifier/files/auth/models.py +29 -0
- package/recipes/backend/auth-verifier/files/auth/settings.py +45 -0
- package/recipes/backend/auth-verifier/files/auth/supabase.py +80 -0
- package/recipes/backend/auth-verifier/files/pytest.ini +5 -0
- package/recipes/backend/auth-verifier/files/tests/__init__.py +0 -0
- package/recipes/backend/auth-verifier/files/tests/test_auth.py +229 -0
- package/recipes/backend/conversation-history/RECIPE.md +49 -0
- package/recipes/backend/conversation-history/files/api/conversations/__init__.py +0 -0
- package/recipes/backend/conversation-history/files/api/conversations/router.py +55 -0
- package/recipes/backend/conversation-history/files/api/conversations/schemas.py +57 -0
- package/recipes/backend/conversation-history/files/pytest.ini +5 -0
- package/recipes/backend/conversation-history/files/tests/test_conversations_api.py +127 -0
- package/recipes/backend/fastapi-base/RECIPE.md +57 -0
- package/recipes/backend/fastapi-base/files/api/app.py +37 -0
- package/recipes/backend/fastapi-base/files/api/base.py +54 -0
- package/recipes/backend/fastapi-base/files/api/errors.py +31 -0
- package/recipes/backend/fastapi-base/files/api/health.py +19 -0
- package/recipes/backend/fastapi-base/files/api/logging_setup.py +26 -0
- package/recipes/backend/fastapi-base/files/api/request_id.py +62 -0
- package/recipes/backend/fastapi-base/files/api/settings.py +32 -0
- package/recipes/backend/fastapi-base/files/pytest.ini +5 -0
- package/recipes/backend/fastapi-base/files/tests/__init__.py +0 -0
- package/recipes/backend/fastapi-base/files/tests/test_api_base.py +136 -0
- package/recipes/backend/observability/RECIPE.md +68 -0
- package/recipes/backend/observability/files/observability/__init__.py +34 -0
- package/recipes/backend/observability/files/observability/factory.py +73 -0
- package/recipes/backend/observability/files/observability/langfuse_tracer.py +29 -0
- package/recipes/backend/observability/files/observability/langsmith_tracer.py +23 -0
- package/recipes/backend/observability/files/observability/models.py +90 -0
- package/recipes/backend/observability/files/observability/opik_tracer.py +30 -0
- package/recipes/backend/observability/files/observability/settings.py +58 -0
- package/recipes/backend/observability/files/observability/tracer.py +142 -0
- package/recipes/backend/observability/files/observability/usage.py +61 -0
- package/recipes/backend/observability/files/pytest.ini +5 -0
- package/recipes/backend/observability/files/tests/__init__.py +0 -0
- package/recipes/backend/observability/files/tests/test_observability.py +219 -0
- package/recipes/backend/project-settings/RECIPE.md +40 -0
- package/recipes/backend/project-settings/files/.env.example +7 -0
- package/recipes/backend/project-settings/files/core/__init__.py +0 -0
- package/recipes/backend/project-settings/files/core/config.py +41 -0
- package/recipes/backend/project-settings/files/pytest.ini +3 -0
- package/recipes/backend/project-settings/files/tests/__init__.py +0 -0
- package/recipes/backend/project-settings/files/tests/test_settings.py +44 -0
- package/recipes/backend/rag-retriever/RECIPE.md +79 -0
- package/recipes/backend/rag-retriever/files/pytest.ini +5 -0
- package/recipes/backend/rag-retriever/files/rag/__init__.py +53 -0
- package/recipes/backend/rag-retriever/files/rag/base.py +106 -0
- package/recipes/backend/rag-retriever/files/rag/chunking.py +84 -0
- package/recipes/backend/rag-retriever/files/rag/document_search.py +60 -0
- package/recipes/backend/rag-retriever/files/rag/embeddings.py +91 -0
- package/recipes/backend/rag-retriever/files/rag/factory.py +82 -0
- package/recipes/backend/rag-retriever/files/rag/in_memory.py +52 -0
- package/recipes/backend/rag-retriever/files/rag/models.py +107 -0
- package/recipes/backend/rag-retriever/files/rag/pgvector.py +138 -0
- package/recipes/backend/rag-retriever/files/rag/qdrant.py +96 -0
- package/recipes/backend/rag-retriever/files/rag/settings.py +48 -0
- package/recipes/backend/rag-retriever/files/tests/__init__.py +0 -0
- package/recipes/backend/rag-retriever/files/tests/rag_conformance.py +95 -0
- package/recipes/backend/rag-retriever/files/tests/test_rag.py +270 -0
- package/recipes/backend/rate-limit-quota/RECIPE.md +77 -0
- package/recipes/backend/rate-limit-quota/files/api/agent/guards.py +62 -0
- package/recipes/backend/rate-limit-quota/files/api/limits/__init__.py +0 -0
- package/recipes/backend/rate-limit-quota/files/api/limits/router.py +38 -0
- package/recipes/backend/rate-limit-quota/files/limits/__init__.py +43 -0
- package/recipes/backend/rate-limit-quota/files/limits/base.py +161 -0
- package/recipes/backend/rate-limit-quota/files/limits/factory.py +69 -0
- package/recipes/backend/rate-limit-quota/files/limits/in_memory.py +36 -0
- package/recipes/backend/rate-limit-quota/files/limits/models.py +108 -0
- package/recipes/backend/rate-limit-quota/files/limits/settings.py +32 -0
- package/recipes/backend/rate-limit-quota/files/limits/sql.py +94 -0
- package/recipes/backend/rate-limit-quota/files/pytest.ini +5 -0
- package/recipes/backend/rate-limit-quota/files/tests/limits_conformance.py +146 -0
- package/recipes/backend/rate-limit-quota/files/tests/test_limits.py +167 -0
- package/recipes/backend/rate-limit-quota/files/tests/test_limits_api.py +100 -0
- package/recipes/backend/voice-providers/RECIPE.md +73 -0
- package/recipes/backend/voice-providers/files/api/__init__.py +0 -0
- package/recipes/backend/voice-providers/files/api/voice/__init__.py +0 -0
- package/recipes/backend/voice-providers/files/api/voice/router.py +113 -0
- package/recipes/backend/voice-providers/files/pytest.ini +5 -0
- package/recipes/backend/voice-providers/files/tests/__init__.py +0 -0
- package/recipes/backend/voice-providers/files/tests/fake_openai.py +44 -0
- package/recipes/backend/voice-providers/files/tests/test_voice.py +164 -0
- package/recipes/backend/voice-providers/files/tests/test_voice_api.py +106 -0
- package/recipes/backend/voice-providers/files/tests/voice_conformance.py +80 -0
- package/recipes/backend/voice-providers/files/voice/__init__.py +29 -0
- package/recipes/backend/voice-providers/files/voice/base.py +116 -0
- package/recipes/backend/voice-providers/files/voice/factory.py +48 -0
- package/recipes/backend/voice-providers/files/voice/fake.py +51 -0
- package/recipes/backend/voice-providers/files/voice/models.py +85 -0
- package/recipes/backend/voice-providers/files/voice/openai_provider.py +116 -0
- package/recipes/backend/voice-providers/files/voice/settings.py +48 -0
- package/recipes/frontend/.gitkeep +0 -0
- package/recipes/frontend/playwright-e2e/RECIPE.md +48 -0
- package/recipes/frontend/playwright-e2e/files/.github/workflows/frontend.yml +35 -0
- package/recipes/frontend/playwright-e2e/files/e2e/helpers.ts +25 -0
- package/recipes/frontend/playwright-e2e/files/e2e/home.spec.ts +10 -0
- package/recipes/frontend/playwright-e2e/files/e2e/login.spec.ts +26 -0
- package/recipes/frontend/playwright-e2e/files/e2e/navigation.spec.ts +27 -0
- package/recipes/frontend/playwright-e2e/files/playwright.config.ts +35 -0
- package/recipes/frontend/react-agent-chat/RECIPE.md +69 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/chat-api.ts +27 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/chat.css +18 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/chat.test.tsx +139 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/components/ChatView.tsx +22 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/components/Composer.tsx +40 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/components/Markdown.tsx +20 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/components/MessageList.tsx +42 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/events.ts +42 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/fake-chat-api.ts +44 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/index.ts +6 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/sse-parser.test.ts +97 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/sse-parser.ts +85 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/use-chat.ts +136 -0
- package/recipes/frontend/react-app-router/RECIPE.md +58 -0
- package/recipes/frontend/react-app-router/files/src/App.test.tsx +54 -0
- package/recipes/frontend/react-app-router/files/src/App.tsx +14 -0
- package/recipes/frontend/react-app-router/files/src/router/ErrorScreen.tsx +21 -0
- package/recipes/frontend/react-app-router/files/src/router/Layout.tsx +28 -0
- package/recipes/frontend/react-app-router/files/src/router/NotFoundScreen.tsx +12 -0
- package/recipes/frontend/react-app-router/files/src/router/index.ts +1 -0
- package/recipes/frontend/react-app-router/files/src/router/router.css +29 -0
- package/recipes/frontend/react-app-router/files/src/router/router.test.tsx +65 -0
- package/recipes/frontend/react-app-router/files/src/router/routes.tsx +59 -0
- package/recipes/frontend/react-app-router/files/src/screens/SettingsScreen.tsx +10 -0
- package/recipes/frontend/react-supabase-auth/RECIPE.md +77 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/LoginScreen.tsx +64 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/SignOutButton.tsx +12 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/auth.css +33 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/auth.test.ts +149 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/auth.ts +59 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/fake-auth.ts +87 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/fake-gotrue.ts +85 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/index.ts +7 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/make-auth.ts +20 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/session-context.tsx +35 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/session-store.ts +63 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/session.test.tsx +56 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/settings.ts +30 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/supabase-auth.ts +85 -0
- package/recipes/frontend/vite-react-base/RECIPE.md +69 -0
- package/recipes/frontend/vite-react-base/files/.env +6 -0
- package/recipes/frontend/vite-react-base/files/.env.fake +2 -0
- package/recipes/frontend/vite-react-base/files/index.html +12 -0
- package/recipes/frontend/vite-react-base/files/package.json +30 -0
- package/recipes/frontend/vite-react-base/files/src/App.test.tsx +49 -0
- package/recipes/frontend/vite-react-base/files/src/App.tsx +13 -0
- package/recipes/frontend/vite-react-base/files/src/api/api.ts +37 -0
- package/recipes/frontend/vite-react-base/files/src/api/fake/fake-api.ts +29 -0
- package/recipes/frontend/vite-react-base/files/src/api/http-api.ts +16 -0
- package/recipes/frontend/vite-react-base/files/src/api/http-client.test.ts +84 -0
- package/recipes/frontend/vite-react-base/files/src/api/http-client.ts +94 -0
- package/recipes/frontend/vite-react-base/files/src/api/make-api.ts +30 -0
- package/recipes/frontend/vite-react-base/files/src/api/types.ts +7 -0
- package/recipes/frontend/vite-react-base/files/src/app-context.tsx +24 -0
- package/recipes/frontend/vite-react-base/files/src/auth/token-provider.ts +27 -0
- package/recipes/frontend/vite-react-base/files/src/config.ts +18 -0
- package/recipes/frontend/vite-react-base/files/src/hooks/use-load.ts +46 -0
- package/recipes/frontend/vite-react-base/files/src/main.tsx +27 -0
- package/recipes/frontend/vite-react-base/files/src/screens/HomeScreen.tsx +28 -0
- package/recipes/frontend/vite-react-base/files/src/styles/global.css +25 -0
- package/recipes/frontend/vite-react-base/files/src/styles/tokens.css +33 -0
- package/recipes/frontend/vite-react-base/files/src/styles/tokens.test.ts +45 -0
- package/recipes/frontend/vite-react-base/files/src/test-setup.ts +8 -0
- package/recipes/frontend/vite-react-base/files/src/vite-env.d.ts +11 -0
- package/recipes/frontend/vite-react-base/files/tsconfig.json +18 -0
- package/recipes/frontend/vite-react-base/files/vite.config.ts +19 -0
- package/unpacked-files.json +3 -0
|
@@ -0,0 +1,219 @@
|
|
|
1
|
+
import logging
|
|
2
|
+
import time
|
|
3
|
+
|
|
4
|
+
import pytest
|
|
5
|
+
from langchain_core.callbacks import BaseCallbackHandler
|
|
6
|
+
from langchain_core.language_models.fake_chat_models import GenericFakeChatModel
|
|
7
|
+
from langchain_core.messages import AIMessage, HumanMessage
|
|
8
|
+
from pydantic import SecretStr
|
|
9
|
+
|
|
10
|
+
from observability import (
|
|
11
|
+
ModelPrice,
|
|
12
|
+
NullTracer,
|
|
13
|
+
ObservabilitySettings,
|
|
14
|
+
Prices,
|
|
15
|
+
RunInfo,
|
|
16
|
+
TokenUsage,
|
|
17
|
+
Tracer,
|
|
18
|
+
TracingFactory,
|
|
19
|
+
TracingProvider,
|
|
20
|
+
UsageCollector,
|
|
21
|
+
usage_from_messages,
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
RUN = RunInfo(thread_id="t1:conv-1", tenant_id="t1", user_id="u1", request_id="req-1", tags=["test"])
|
|
25
|
+
PRICES = Prices(by_model={"gpt-x": ModelPrice(input=2.0, output=8.0, cached_input=0.5)})
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def ai(model: str, *, inp: int, out: int, cached: int = 0, text: str = "ok") -> AIMessage:
|
|
29
|
+
return AIMessage(
|
|
30
|
+
text,
|
|
31
|
+
usage_metadata={"input_tokens": inp, "output_tokens": out, "total_tokens": inp + out, "input_token_details": {"cache_read": cached}},
|
|
32
|
+
response_metadata={"model_name": model},
|
|
33
|
+
)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
# --- usage and cost -------------------------------------------------------------------------
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def test_usage_from_messages_reads_usage_metadata_and_cache():
|
|
40
|
+
usages = usage_from_messages([HumanMessage("q"), ai("gpt-x", inp=100, out=20, cached=40), AIMessage("no usage")])
|
|
41
|
+
assert usages == [TokenUsage(model="gpt-x", input_tokens=100, output_tokens=20, cached_input_tokens=40)]
|
|
42
|
+
assert usages[0].total_tokens == 120
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def test_prices_discount_cached_input_and_refuse_to_guess():
|
|
46
|
+
usage = TokenUsage(model="gpt-x", input_tokens=100, output_tokens=20, cached_input_tokens=40)
|
|
47
|
+
assert PRICES.cost_usd(usage) == pytest.approx((60 * 2.0 + 40 * 0.5 + 20 * 8.0) / 1e6)
|
|
48
|
+
assert PRICES.cost_usd(TokenUsage(model="unknown-model", input_tokens=1)) is None
|
|
49
|
+
assert Prices(by_model={"m": ModelPrice(input=1, output=1)}).cost_usd(TokenUsage(model="m", input_tokens=10, cached_input_tokens=10)) == pytest.approx(10 / 1e6)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
async def test_usage_collector_sees_every_chat_model_call():
|
|
53
|
+
collector = UsageCollector()
|
|
54
|
+
model = GenericFakeChatModel(messages=iter([ai("gpt-x", inp=10, out=5), ai("gpt-x", inp=30, out=1, cached=30)]))
|
|
55
|
+
await model.ainvoke("a", config={"callbacks": [collector]})
|
|
56
|
+
await model.ainvoke("b", config={"callbacks": [collector]})
|
|
57
|
+
assert [u.input_tokens for u in collector.usages] == [10, 30]
|
|
58
|
+
assert (collector.input_tokens, collector.output_tokens, collector.cached_input_tokens) == (40, 6, 30)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
# --- turn trace -----------------------------------------------------------------------------
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
async def test_null_tracer_still_gives_metadata_collector_and_summary(caplog):
|
|
65
|
+
tracer = NullTracer(prices=PRICES)
|
|
66
|
+
trace = tracer.begin(RUN)
|
|
67
|
+
assert trace.config["metadata"] == {"thread_id": "t1:conv-1", "tenant_id": "t1", "user_id": "u1", "request_id": "req-1"}
|
|
68
|
+
assert trace.config["tags"] == ["test"] and trace.config["run_name"] == "turn"
|
|
69
|
+
assert trace.config["callbacks"] == [trace.usage]
|
|
70
|
+
|
|
71
|
+
model = GenericFakeChatModel(messages=iter([ai("gpt-x", inp=100, out=20, cached=40)]))
|
|
72
|
+
await model.ainvoke("q", config=trace.config)
|
|
73
|
+
time.sleep(0.01)
|
|
74
|
+
with caplog.at_level(logging.INFO, logger="observability.tracer"):
|
|
75
|
+
summary = trace.finish()
|
|
76
|
+
assert (summary.llm_calls, summary.input_tokens, summary.output_tokens, summary.cached_input_tokens) == (1, 100, 20, 40)
|
|
77
|
+
assert summary.cost_usd == pytest.approx(0.00030) and summary.duration_ms >= 10 and summary.error is None
|
|
78
|
+
assert "turn thread=t1:conv-1 tenant=t1 llm_calls=1 tokens=100/20 cached=40 cost_usd=0.000300" in caplog.text
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def test_finish_records_the_error_class_only_and_unknown_cost():
|
|
82
|
+
trace = NullTracer().begin(RUN)
|
|
83
|
+
trace.usage.usages.append(TokenUsage(model="no-price", input_tokens=5))
|
|
84
|
+
summary = trace.finish(error=RuntimeError("postgres://user:secret@host"))
|
|
85
|
+
assert summary.error == "RuntimeError" and summary.cost_usd is None and "secret" not in summary.model_dump_json()
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
class Spy(BaseCallbackHandler):
|
|
89
|
+
def __init__(self) -> None:
|
|
90
|
+
self.starts: list[dict] = []
|
|
91
|
+
|
|
92
|
+
def on_chat_model_start(self, serialized, messages, *, metadata=None, tags=None, **kwargs):
|
|
93
|
+
self.starts.append({"metadata": dict(metadata or {}), "tags": list(tags or [])})
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
class SpyTracer(Tracer):
|
|
97
|
+
def __init__(self) -> None:
|
|
98
|
+
super().__init__()
|
|
99
|
+
self.spy = Spy()
|
|
100
|
+
self.flushed = 0
|
|
101
|
+
|
|
102
|
+
def _callbacks(self, run):
|
|
103
|
+
return [self.spy]
|
|
104
|
+
|
|
105
|
+
def _metadata(self, run):
|
|
106
|
+
return {"provider_session": run.thread_id}
|
|
107
|
+
|
|
108
|
+
def _trace_id(self, run):
|
|
109
|
+
return f"trace-{run.request_id}"
|
|
110
|
+
|
|
111
|
+
def _flush(self):
|
|
112
|
+
self.flushed += 1
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
async def test_provider_handler_receives_metadata_and_tags_of_the_run():
|
|
116
|
+
tracer = SpyTracer()
|
|
117
|
+
trace = tracer.begin(RUN)
|
|
118
|
+
await GenericFakeChatModel(messages=iter([ai("gpt-x", inp=1, out=1)])).ainvoke("q", config=trace.config)
|
|
119
|
+
assert tracer.spy.starts[0]["metadata"]["provider_session"] == "t1:conv-1"
|
|
120
|
+
assert tracer.spy.starts[0]["metadata"]["request_id"] == "req-1" and "test" in tracer.spy.starts[0]["tags"]
|
|
121
|
+
assert trace.finish().trace_id == "trace-req-1"
|
|
122
|
+
await tracer.flush()
|
|
123
|
+
assert tracer.flushed == 1
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
class BrokenTracer(Tracer):
|
|
127
|
+
def _callbacks(self, run):
|
|
128
|
+
raise ConnectionError("sk-secret inside the message")
|
|
129
|
+
|
|
130
|
+
def _flush(self):
|
|
131
|
+
raise ConnectionError("sk-secret inside the message")
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
class SlowTracer(NullTracer):
|
|
135
|
+
def _flush(self):
|
|
136
|
+
time.sleep(2)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
async def test_tracing_failures_never_reach_the_turn(caplog):
|
|
140
|
+
with caplog.at_level(logging.WARNING, logger="observability.tracer"):
|
|
141
|
+
trace = BrokenTracer().begin(RUN)
|
|
142
|
+
await BrokenTracer().flush()
|
|
143
|
+
await SlowTracer(flush_timeout=0.05).flush()
|
|
144
|
+
assert trace.config["callbacks"] == [trace.usage] and trace.config["metadata"]["thread_id"] == "t1:conv-1"
|
|
145
|
+
assert "tracing disabled for this run (ConnectionError)" in caplog.text
|
|
146
|
+
assert "tracing flush failed (ConnectionError)" in caplog.text and "flush timed out" in caplog.text
|
|
147
|
+
assert "sk-secret" not in caplog.text
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
# --- settings and factory -------------------------------------------------------------------
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def test_settings_default_to_no_tracing_and_check_each_provider(monkeypatch):
|
|
154
|
+
assert ObservabilitySettings(_env_file=None).tracing_provider is TracingProvider.NONE
|
|
155
|
+
with pytest.raises(ValueError, match="LANGFUSE_PUBLIC_KEY"):
|
|
156
|
+
ObservabilitySettings(_env_file=None, tracing_provider="langfuse")
|
|
157
|
+
with pytest.raises(ValueError, match="OPIK_URL_OVERRIDE"):
|
|
158
|
+
ObservabilitySettings(_env_file=None, tracing_provider="opik")
|
|
159
|
+
with pytest.raises(ValueError, match="LANGSMITH_API_KEY"):
|
|
160
|
+
ObservabilitySettings(_env_file=None, tracing_provider="langsmith")
|
|
161
|
+
monkeypatch.setenv("TRACING_TAGS", "prod, eu")
|
|
162
|
+
monkeypatch.setenv("TRACING_PROVIDER", "opik")
|
|
163
|
+
monkeypatch.setenv("OPIK_URL_OVERRIDE", "http://localhost:5173/api")
|
|
164
|
+
settings = ObservabilitySettings(_env_file=None)
|
|
165
|
+
assert settings.tracing_tags == ["prod", "eu"] and settings.tracing_provider is TracingProvider.OPIK
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
def test_factory_none_and_fail_open(monkeypatch, caplog):
|
|
169
|
+
assert isinstance(TracingFactory.create(ObservabilitySettings(_env_file=None)), NullTracer)
|
|
170
|
+
settings = ObservabilitySettings(_env_file=None, tracing_provider="langsmith", langsmith_api_key=SecretStr("ls-x"))
|
|
171
|
+
monkeypatch.setattr(TracingFactory, "_langsmith", staticmethod(lambda *a, **k: (_ for _ in ()).throw(ImportError("langsmith"))))
|
|
172
|
+
with caplog.at_level(logging.WARNING):
|
|
173
|
+
tracer = TracingFactory.create(settings)
|
|
174
|
+
assert isinstance(tracer, NullTracer) and "langsmith unavailable (ImportError)" in caplog.text
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def test_factory_builds_langsmith_offline():
|
|
178
|
+
pytest.importorskip("langsmith")
|
|
179
|
+
from observability.langsmith_tracer import LangSmithTracer
|
|
180
|
+
|
|
181
|
+
settings = ObservabilitySettings(_env_file=None, tracing_provider="langsmith", langsmith_api_key=SecretStr("ls-x"), langsmith_project="p")
|
|
182
|
+
tracer = TracingFactory.create(settings)
|
|
183
|
+
assert isinstance(tracer, LangSmithTracer) and tracer.project_name == "p"
|
|
184
|
+
callbacks = tracer.begin(RUN).config["callbacks"]
|
|
185
|
+
assert type(callbacks[0]).__name__ == "LangChainTracer" and isinstance(callbacks[1], UsageCollector)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def test_factory_builds_langfuse_offline():
|
|
189
|
+
pytest.importorskip("langfuse")
|
|
190
|
+
from observability.langfuse_tracer import LangfuseTracer
|
|
191
|
+
|
|
192
|
+
settings = ObservabilitySettings(
|
|
193
|
+
_env_file=None,
|
|
194
|
+
tracing_provider="langfuse",
|
|
195
|
+
langfuse_public_key=SecretStr("pk-lf-test"),
|
|
196
|
+
langfuse_secret_key=SecretStr("sk-lf-test"),
|
|
197
|
+
langfuse_host="http://127.0.0.1:9",
|
|
198
|
+
)
|
|
199
|
+
tracer = TracingFactory.create(settings)
|
|
200
|
+
assert isinstance(tracer, LangfuseTracer)
|
|
201
|
+
config = tracer.begin(RUN).config
|
|
202
|
+
assert isinstance(config["callbacks"][0], BaseCallbackHandler) and "langfuse" in type(config["callbacks"][0]).__module__
|
|
203
|
+
assert config["metadata"]["langfuse_session_id"] == "t1:conv-1" and config["metadata"]["langfuse_user_id"] == "u1"
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def test_factory_builds_opik_offline(monkeypatch):
|
|
207
|
+
opik = pytest.importorskip("opik")
|
|
208
|
+
from observability.opik_tracer import OpikTracer
|
|
209
|
+
|
|
210
|
+
configured: list[dict] = []
|
|
211
|
+
monkeypatch.setattr(opik, "configure", lambda **kwargs: configured.append(kwargs))
|
|
212
|
+
monkeypatch.setenv("OPIK_URL_OVERRIDE", "http://127.0.0.1:9/api")
|
|
213
|
+
settings = ObservabilitySettings(_env_file=None, tracing_provider="opik", opik_url_override="http://127.0.0.1:9/api", opik_project_name="p")
|
|
214
|
+
tracer = TracingFactory.create(settings)
|
|
215
|
+
assert isinstance(tracer, OpikTracer) and configured == [
|
|
216
|
+
{"automatic_approvals": True, "install_mcp": False, "use_local": True, "url_override": "http://127.0.0.1:9/api"}
|
|
217
|
+
]
|
|
218
|
+
callbacks = tracer.begin(RUN).config["callbacks"]
|
|
219
|
+
assert type(callbacks[0]).__name__ == "OpikTracer" and tracer.project_name == "p"
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# project-settings
|
|
2
|
+
|
|
3
|
+
## Cuándo usarla
|
|
4
|
+
|
|
5
|
+
Cuando un backend en Python necesite **una sola configuración para todo el proyecto**, cargada desde variables de entorno y `.env`. Es la base sobre la que otras recetas añaden su bloque. Por ejemplo, `langchain-agent` añade `LLMSettings`.
|
|
6
|
+
|
|
7
|
+
No depende de ninguna otra receta.
|
|
8
|
+
|
|
9
|
+
## Qué incluye
|
|
10
|
+
|
|
11
|
+
```
|
|
12
|
+
core/
|
|
13
|
+
config.py BaseAppSettings (campos comunes) + Settings (compone bloques) + get_settings() en caché
|
|
14
|
+
.env.example variables base; cada bloque añade las suyas
|
|
15
|
+
tests/
|
|
16
|
+
test_settings.py 4 tests sobre BaseAppSettings: env > defaults, caché, composición, valores explícitos > env
|
|
17
|
+
pytest.ini
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
## Decisiones
|
|
21
|
+
|
|
22
|
+
- **Un único `Settings` por proyecto**, en la raíz del backend (`core/config.py`), al lado de `api/`, `db/` y `agentic/`. Nadie lee `os.environ` directamente.
|
|
23
|
+
- **pydantic-settings.** Las variables de entorno no distinguen mayúsculas (`APP_NAME` → `app_name`) y tienen prioridad sobre el `.env`. Con `extra="ignore"`, las variables que sobran no rompen nada.
|
|
24
|
+
- **`get_settings()` con `lru_cache`.** Se carga una vez. En tests: `get_settings.cache_clear()` o `Settings(_env_file=None, ...)`.
|
|
25
|
+
- **Composición por bloques.** Cada módulo con configuración propia trae su `BaseSettings` y `Settings` lo hereda: `class Settings(LLMSettings, ObservabilitySettings, BaseAppSettings)`. Esta receta no conoce esos bloques. Son ellos los que hacen referencia a esta.
|
|
26
|
+
- **Secretos en `SecretStr`**, para que no salgan en los logs ni en el `repr`.
|
|
27
|
+
|
|
28
|
+
## Cómo adaptarla
|
|
29
|
+
|
|
30
|
+
1. Copia `files/` a la raíz del backend. Si `core/` va dentro de un paquete (`app/core/`), cambia los imports de los tests.
|
|
31
|
+
2. Sustituye `{{app_name}}` en `core/config.py` y `.env.example`.
|
|
32
|
+
3. `cp .env.example .env`. No subas el `.env` a git: añádelo a `.gitignore`.
|
|
33
|
+
4. **Añadir un bloque de otra receta:** impórtalo en `core/config.py` y ponlo como padre de `Settings`, delante de `BaseAppSettings`. Añade al `.env` sus variables, que vienen en el RECIPE.md de esa receta.
|
|
34
|
+
5. Para usarlo: `from core.config import get_settings`; `get_settings().environment`.
|
|
35
|
+
|
|
36
|
+
**Dependencias:** `pydantic-settings>=2.6` y `pytest` para los tests.
|
|
37
|
+
|
|
38
|
+
## Origen
|
|
39
|
+
|
|
40
|
+
- Probado en producción: `Settings(BaseSettings)` con `.env` en la raíz del backend, `get_settings()` con `lru_cache` (en otros proyectos, singleton de módulo) y composición heredando bloques (`Settings(ObservabilitySettings)`).
|
|
File without changes
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
"""Project settings: ONE object for the whole backend, loaded from env vars / `.env`.
|
|
2
|
+
|
|
3
|
+
Lives at the backend root (next to `api/`, `db/`, `agentic/`...). Everyone reads it through
|
|
4
|
+
`get_settings()`; nobody calls `os.environ` directly.
|
|
5
|
+
|
|
6
|
+
Composition: each module that needs config brings its own block (a `BaseSettings` subclass)
|
|
7
|
+
and `Settings` inherits it. This recipe knows none of them; you add them here:
|
|
8
|
+
|
|
9
|
+
from agentic.llm.settings import LLMSettings # block from the langchain-agent recipe
|
|
10
|
+
|
|
11
|
+
class Settings(LLMSettings, BaseAppSettings): ...
|
|
12
|
+
|
|
13
|
+
- Field names are case-insensitive env vars: `app_name` <- `APP_NAME`.
|
|
14
|
+
- Real env vars win over `.env` (production injects them; `.env` is for local dev).
|
|
15
|
+
- Secrets are `SecretStr`: they never show up in logs or `repr`.
|
|
16
|
+
- Tests: `Settings(_env_file=None, ...)`, or `get_settings.cache_clear()` after changing env vars.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from functools import lru_cache
|
|
20
|
+
|
|
21
|
+
from pydantic_settings import BaseSettings, SettingsConfigDict
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class BaseAppSettings(BaseSettings):
|
|
25
|
+
"""Fields every backend has. Blocks from other recipes are mixed in by `Settings`."""
|
|
26
|
+
|
|
27
|
+
model_config = SettingsConfigDict(env_file=".env", env_file_encoding="utf-8", extra="ignore")
|
|
28
|
+
|
|
29
|
+
app_name: str = "{{app_name}}"
|
|
30
|
+
environment: str = "local" # local | staging | production
|
|
31
|
+
debug: bool = False
|
|
32
|
+
log_level: str = "INFO"
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class Settings(BaseAppSettings):
|
|
36
|
+
"""The project's settings. Add blocks as parents: `class Settings(LLMSettings, BaseAppSettings)`."""
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
@lru_cache(maxsize=1)
|
|
40
|
+
def get_settings() -> Settings:
|
|
41
|
+
return Settings()
|
|
File without changes
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
"""Tests of the base behaviour. They use `BaseAppSettings`, not `Settings`, so they keep passing
|
|
2
|
+
after blocks with required fields (e.g. MEMORY_BACKEND) are mixed into `Settings`."""
|
|
3
|
+
|
|
4
|
+
from pydantic import SecretStr
|
|
5
|
+
from pydantic_settings import BaseSettings
|
|
6
|
+
|
|
7
|
+
from core import config
|
|
8
|
+
from core.config import BaseAppSettings
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def test_env_vars_override_defaults(monkeypatch):
|
|
12
|
+
monkeypatch.setenv("ENVIRONMENT", "production")
|
|
13
|
+
monkeypatch.setenv("DEBUG", "true")
|
|
14
|
+
settings = BaseAppSettings(_env_file=None)
|
|
15
|
+
assert settings.environment == "production"
|
|
16
|
+
assert settings.debug is True
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def test_get_settings_is_cached(monkeypatch):
|
|
20
|
+
monkeypatch.setattr(config, "Settings", BaseAppSettings)
|
|
21
|
+
config.get_settings.cache_clear()
|
|
22
|
+
try:
|
|
23
|
+
assert config.get_settings() is config.get_settings()
|
|
24
|
+
finally:
|
|
25
|
+
config.get_settings.cache_clear()
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def test_blocks_compose_by_inheritance(monkeypatch):
|
|
29
|
+
class PaymentsSettings(BaseSettings):
|
|
30
|
+
payments_api_key: SecretStr | None = None
|
|
31
|
+
|
|
32
|
+
class Composed(PaymentsSettings, BaseAppSettings):
|
|
33
|
+
pass
|
|
34
|
+
|
|
35
|
+
monkeypatch.setenv("PAYMENTS_API_KEY", "secret")
|
|
36
|
+
settings = Composed(_env_file=None)
|
|
37
|
+
assert settings.payments_api_key.get_secret_value() == "secret"
|
|
38
|
+
assert "secret" not in repr(settings)
|
|
39
|
+
assert settings.app_name
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def test_init_values_win_over_env(monkeypatch):
|
|
43
|
+
monkeypatch.setenv("LOG_LEVEL", "DEBUG")
|
|
44
|
+
assert BaseAppSettings(_env_file=None, log_level="WARNING").log_level == "WARNING"
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
# rag-retriever
|
|
2
|
+
|
|
3
|
+
## Cuándo usarla
|
|
4
|
+
|
|
5
|
+
Para **que el agente busque en los documentos del tenant**: trocear, embeber, guardar los vectores en un store elegido en `.env` (RAM, Qdrant o Postgres con pgvector) y buscar por similitud con aislamiento por tenant. Trae el servicio que la tool `search_documents` de `langchain-agent` espera (`document_search`), así que encaja en `agent-chat-api` sin tocar el agente.
|
|
6
|
+
|
|
7
|
+
No depende de otras recetas. **Usa `project-settings`** (bloque `RagSettings`) y **encaja con `langchain-agent`** (`DocumentSearchService`). No trae endpoints de ingesta ni lectura de PDF: tú conviertes el archivo en texto y llamas a `retriever.index`.
|
|
8
|
+
|
|
9
|
+
## Qué incluye
|
|
10
|
+
|
|
11
|
+
```
|
|
12
|
+
rag/
|
|
13
|
+
models.py Document · Chunk (id uuid5 + content_hash) · EmbeddedChunk · SearchHit · IndexResult · RetrievalError · RagBackend · EmbeddingsProvider
|
|
14
|
+
chunking.py TextChunker(size, overlap): párrafos → líneas → frases → palabras, con solape en límite de palabra
|
|
15
|
+
embeddings.py Embedder (ABC, errores → RetrievalError) · OpenAIEmbedder (text-embedding-3-*, lotes) · HashingEmbedder (local, sin red)
|
|
16
|
+
base.py VectorStore (ABC: upsert, existing_hashes, delete_chunks, delete_document, search) · Retriever (index, remove, search)
|
|
17
|
+
in_memory.py InMemoryVectorStore: referencia y desarrollo
|
|
18
|
+
qdrant.py QdrantVectorStore: una colección, payload = chunk, índices keyword en tenant_id y document_id
|
|
19
|
+
pgvector.py PgVectorStore: tabla rag_chunks, coseno <=>, HNSW; schema_sql(dimension) para tu migración
|
|
20
|
+
settings.py bloque RagSettings: RAG_BACKEND (obligatorio), EMBEDDINGS_PROVIDER, EMBEDDING_*, RAG_CHUNK_*, RAG_SEARCH_LIMIT, RAG_MIN_SCORE, QDRANT_*
|
|
21
|
+
factory.py EmbedderFactory · VectorStoreFactory · RagFactory.create(settings, session_factory=, embedder=, store=)
|
|
22
|
+
document_search.py DocumentSearchService(retriever): el `document_search` de la tool · format_hits(hits): contexto numerado con cita, sin scores
|
|
23
|
+
tests/
|
|
24
|
+
rag_conformance.py VectorStoreConformance: 7 tests que todo backend debe pasar (orden, tenant, filtros, upsert, borrado)
|
|
25
|
+
test_rag.py 28 tests: conformidad en RAM y Qdrant (`:memory:`), chunking, embedders, retriever incremental, servicio, settings, factory
|
|
26
|
+
(+7 de pgvector, saltados salvo RAG_TEST_DATABASE_URL)
|
|
27
|
+
pytest.ini
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
## Decisiones
|
|
31
|
+
|
|
32
|
+
- **Lógica en `Retriever`, almacenamiento en `VectorStore`.** Trocear, decidir qué embeber, borrar lo sobrante y aplicar el umbral es igual en todos los backends; el store solo guarda, filtra y ordena por coseno. Un backend nuevo es una subclase más los tests de conformidad (como `agent-memory` y `rate-limit-quota`).
|
|
33
|
+
- **Store y embeddings elegidos en settings, sin valor por defecto para el store** (`RAG_BACKEND`): caer en RAM en producción perdería el índice en cada reinicio.
|
|
34
|
+
- **Ids deterministas y hash del contenido.** `Chunk.id = uuid5(tenant:documento:índice)` y `content_hash = sha256(texto)`: reindexar el mismo documento no embebe nada, cambiar un párrafo embebe un chunk y acortarlo borra los que sobran. El `Document.id` lo pone la app y debe ser estable (ruta, clave primaria), nunca aleatorio.
|
|
35
|
+
- **Tenant obligatorio y dentro de la consulta.** Todos los métodos lo toman como argumento y los backends lo filtran antes del `LIMIT`. No es un filtro más: `filters` son solo igualdades sobre `Chunk.metadata`.
|
|
36
|
+
- **Coseno en [0, 1] y umbral absoluto opcional** (`RAG_MIN_SCORE`, por defecto 0 = sin umbral). En producción se han usado valores entre 0.4 y 0.5 con `text-embedding-3-small`, o un umbral relativo al mejor resultado. Depende del modelo y del corpus: mídelo con tus datos antes de ponerlo.
|
|
37
|
+
- **Lo que ve el LLM es texto con cita, sin scores ni tenant** (`format_hits`). Los scores van en el resultado del servicio por si la UI los quiere.
|
|
38
|
+
- **`DocumentSearchService` cumple el protocolo de la tool** (`search(tenant_id, query, limit) -> [{title, snippet, ...}]`): la receta `langchain-agent` no cambia. Si prefieres un nodo del grafo, llama a `retriever.search` desde el nodo.
|
|
39
|
+
- **Un solo error, sin texto del fondo.** `RetrievalError(kind, stage)` lleva el nombre de la clase del error original (`ConnectionError`), nunca su mensaje: puede contener la URL de la base de datos o la clave. `AppTool` ya lo convierte en `SAFE_FAILURE` para el LLM.
|
|
40
|
+
- **Chunking por caracteres, recursivo** (1000/200, como `RecursiveCharacterTextSplitter`), sin dependencia de LangChain ni tokenizador. Si necesitas trocear por tokens (por ejemplo 500/50 con tiktoken), sustituye `TextChunker.split`.
|
|
41
|
+
- **`HashingEmbedder` como proveedor local**, no solo un fake de tests: bolsa de palabras con hashing, determinista, cosenos con sentido. Permite arrancar el proyecto y la demo sin clave de OpenAI.
|
|
42
|
+
- **Búsqueda semántica solo.** En producción es habitual añadir una rama léxica (`tsvector`) y fusión RRF con k=50, e incluso un juez LLM. Son extensiones de `PgVectorStore.search` (dos consultas y fusión en Python), no del contrato.
|
|
43
|
+
- **Qdrant se prueba sin servidor** con `AsyncQdrantClient(":memory:")`. pgvector solo contra un Postgres real: sus tests se saltan sin `RAG_TEST_DATABASE_URL`.
|
|
44
|
+
|
|
45
|
+
## Cómo adaptarla
|
|
46
|
+
|
|
47
|
+
1. Copia `files/` en la raíz del backend. Sustituye `{{app_name}}` en `rag/settings.py` (nombre de la colección).
|
|
48
|
+
2. **Settings:** `class Settings(RagSettings, LLMSettings, BaseAppSettings)` (comparte `OPENAI_API_KEY` con `langchain-agent`). En el `.env`:
|
|
49
|
+
|
|
50
|
+
```
|
|
51
|
+
RAG_BACKEND=qdrant # memory | qdrant | pgvector
|
|
52
|
+
QDRANT_URL=https://... QDRANT_API_KEY=...
|
|
53
|
+
EMBEDDINGS_PROVIDER=openai # hashing en local sin clave
|
|
54
|
+
EMBEDDING_MODEL=text-embedding-3-small
|
|
55
|
+
EMBEDDING_DIMENSION=1536
|
|
56
|
+
RAG_MIN_SCORE=0.0
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
Con `pgvector`, ejecuta `schema_sql(settings.embedding_dimension)` en tu migración (o `await store.ensure_ready()` en desarrollo) y pasa `session_factory=SessionLocal` a la factory.
|
|
60
|
+
3. **Arranque** (`lifespan` de `create_app`):
|
|
61
|
+
|
|
62
|
+
```python
|
|
63
|
+
retriever = RagFactory.create(settings, session_factory=SessionLocal)
|
|
64
|
+
await retriever.store.ensure_ready()
|
|
65
|
+
app.state.retriever = retriever
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
y al agente: `registry.configure(document_search=DocumentSearchService(retriever))`.
|
|
69
|
+
4. **Ingesta:** convierte cada archivo en texto (pypdf, pypdfium2…) y llama a `await retriever.index(tenant_id, Document(id=ruta, title=..., text=..., source=url, metadata={"category": ...}))`. Es idempotente: llámalo también cuando el documento cambie. Para borrar, `retriever.remove(tenant_id, document_id)`. Cambiar de modelo o de dimensión obliga a reindexar todo con una colección nueva.
|
|
70
|
+
5. **Prompt sin tool:** `format_hits(await retriever.search(tenant_id, question))` como bloque de contexto.
|
|
71
|
+
6. **Backend nuevo:** subclase de `VectorStore` + `class TestMiStore(VectorStoreConformance)` con la fixture `store`, y un caso en `VectorStoreFactory`.
|
|
72
|
+
7. Para comprobarla: `pytest` (28 tests sin red; con `RAG_TEST_DATABASE_URL=postgresql://...` corren también los 7 de pgvector).
|
|
73
|
+
|
|
74
|
+
**Dependencias:** `pydantic-settings`; `openai` (proveedor openai); `qdrant-client` (backend qdrant, probado con 1.20); `sqlalchemy` + `psycopg` y la extensión `vector` en Postgres (backend pgvector).
|
|
75
|
+
|
|
76
|
+
## Origen
|
|
77
|
+
|
|
78
|
+
- Probado en producción: store elegido por settings con clase base y factory; ids deterministas `uuid5` para upsert idempotente y hash del contenido para embeber solo lo cambiado; tenant obligatorio filtrado dentro de la consulta; `Embedder` como contrato con doble local y `RetrievalError` sin texto del fondo; chunking recursivo por caracteres 1000/200; contexto numerado con fuente y sin scores; pgvector con coseno `<=>` y HNSW.
|
|
79
|
+
- Nuevo en esta receta: `HashingEmbedder` como proveedor local con cosenos con sentido (no solo un fake de tests) y la batería de conformidad `VectorStoreConformance` para cualquier backend.
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
"""Retrieval layer: documents -> chunks -> vectors in a store chosen by settings -> search by tenant.
|
|
2
|
+
|
|
3
|
+
retriever = RagFactory.create(settings, session_factory=SessionLocal) # startup
|
|
4
|
+
await retriever.store.ensure_ready()
|
|
5
|
+
await retriever.index("tenant-1", Document(id="manual.pdf", title="Manual", text=...))
|
|
6
|
+
hits = await retriever.search("tenant-1", "how do I reset it?")
|
|
7
|
+
ToolSet.default(document_search=DocumentSearchService(retriever)) # agent tool
|
|
8
|
+
|
|
9
|
+
Backends do not import each other: `rag/` depends on `qdrant-client` or SQLAlchemy only through
|
|
10
|
+
the backend you pick in the factory.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from .base import Retriever, VectorStore
|
|
14
|
+
from .chunking import TextChunker
|
|
15
|
+
from .document_search import DocumentSearchService, format_hits
|
|
16
|
+
from .embeddings import Embedder, HashingEmbedder, OpenAIEmbedder
|
|
17
|
+
from .factory import EmbedderFactory, RagFactory, VectorStoreFactory
|
|
18
|
+
from .models import (
|
|
19
|
+
Chunk,
|
|
20
|
+
Document,
|
|
21
|
+
EmbeddedChunk,
|
|
22
|
+
EmbeddingsProvider,
|
|
23
|
+
Filters,
|
|
24
|
+
IndexResult,
|
|
25
|
+
RagBackend,
|
|
26
|
+
RetrievalError,
|
|
27
|
+
SearchHit,
|
|
28
|
+
)
|
|
29
|
+
from .settings import RagSettings
|
|
30
|
+
|
|
31
|
+
__all__ = [
|
|
32
|
+
"Chunk",
|
|
33
|
+
"Document",
|
|
34
|
+
"DocumentSearchService",
|
|
35
|
+
"EmbeddedChunk",
|
|
36
|
+
"Embedder",
|
|
37
|
+
"EmbedderFactory",
|
|
38
|
+
"EmbeddingsProvider",
|
|
39
|
+
"Filters",
|
|
40
|
+
"HashingEmbedder",
|
|
41
|
+
"IndexResult",
|
|
42
|
+
"OpenAIEmbedder",
|
|
43
|
+
"RagBackend",
|
|
44
|
+
"RagFactory",
|
|
45
|
+
"RagSettings",
|
|
46
|
+
"RetrievalError",
|
|
47
|
+
"Retriever",
|
|
48
|
+
"SearchHit",
|
|
49
|
+
"TextChunker",
|
|
50
|
+
"VectorStore",
|
|
51
|
+
"VectorStoreFactory",
|
|
52
|
+
"format_hits",
|
|
53
|
+
]
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
"""`VectorStore`: the contract every backend implements. `Retriever`: the logic on top of it
|
|
2
|
+
(chunk, hash, embed only what changed, drop stale chunks, search with a threshold), which is the
|
|
3
|
+
same whatever the backend. A new backend is a `VectorStore` subclass plus the conformance tests.
|
|
4
|
+
|
|
5
|
+
Tenant isolation is not optional: every method takes `tenant_id` and the backends apply it inside
|
|
6
|
+
the query, before any limit. `filters` are equality filters on `Chunk.metadata`.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from abc import ABC, abstractmethod
|
|
10
|
+
|
|
11
|
+
from .chunking import TextChunker
|
|
12
|
+
from .embeddings import Embedder
|
|
13
|
+
from .models import Document, EmbeddedChunk, Filters, IndexResult, RetrievalError, SearchHit
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class VectorStore(ABC):
|
|
17
|
+
"""Storage only. No embedding, no chunking, no scoring policy: that lives in `Retriever`."""
|
|
18
|
+
|
|
19
|
+
async def ensure_ready(self) -> None:
|
|
20
|
+
"""Create the collection/table if missing. Idempotent; call it once at startup."""
|
|
21
|
+
|
|
22
|
+
@abstractmethod
|
|
23
|
+
async def upsert(self, chunks: list[EmbeddedChunk]) -> None:
|
|
24
|
+
"""Insert or replace by `chunk.id`."""
|
|
25
|
+
|
|
26
|
+
@abstractmethod
|
|
27
|
+
async def existing_hashes(self, tenant_id: str, document_id: str) -> dict[str, str]:
|
|
28
|
+
"""`{chunk_id: content_hash}` of what is stored for the document (empty if unknown)."""
|
|
29
|
+
|
|
30
|
+
@abstractmethod
|
|
31
|
+
async def delete_chunks(self, tenant_id: str, chunk_ids: list[str]) -> None: ...
|
|
32
|
+
|
|
33
|
+
@abstractmethod
|
|
34
|
+
async def delete_document(self, tenant_id: str, document_id: str) -> int:
|
|
35
|
+
"""Remove every chunk of the document. Returns how many were removed."""
|
|
36
|
+
|
|
37
|
+
@abstractmethod
|
|
38
|
+
async def search(
|
|
39
|
+
self, tenant_id: str, vector: list[float], *, limit: int, filters: Filters | None = None
|
|
40
|
+
) -> list[SearchHit]:
|
|
41
|
+
"""Top `limit` chunks of the tenant by cosine similarity, best first, no threshold."""
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class Retriever:
|
|
45
|
+
"""What the app uses. Built by `RagFactory.create(settings, ...)`."""
|
|
46
|
+
|
|
47
|
+
def __init__(
|
|
48
|
+
self,
|
|
49
|
+
*,
|
|
50
|
+
embedder: Embedder,
|
|
51
|
+
store: VectorStore,
|
|
52
|
+
chunker: TextChunker | None = None,
|
|
53
|
+
limit: int = 5,
|
|
54
|
+
min_score: float = 0.0,
|
|
55
|
+
) -> None:
|
|
56
|
+
self.embedder = embedder
|
|
57
|
+
self.store = store
|
|
58
|
+
self.chunker = chunker or TextChunker()
|
|
59
|
+
self.limit = limit
|
|
60
|
+
self.min_score = min_score
|
|
61
|
+
|
|
62
|
+
async def index(self, tenant_id: str, document: Document) -> IndexResult:
|
|
63
|
+
"""Idempotent: same text -> nothing embedded; one paragraph changed -> one chunk embedded;
|
|
64
|
+
shorter document -> the chunks past the end are deleted."""
|
|
65
|
+
chunks = self.chunker.chunk(tenant_id, document)
|
|
66
|
+
stored = await self._guard(self.store.existing_hashes(tenant_id, document.id), stage="store")
|
|
67
|
+
changed = [chunk for chunk in chunks if stored.get(chunk.id) != chunk.content_hash]
|
|
68
|
+
if changed:
|
|
69
|
+
vectors = await self.embedder.embed([chunk.text for chunk in changed])
|
|
70
|
+
embedded = [EmbeddedChunk(chunk=chunk, vector=vector) for chunk, vector in zip(changed, vectors, strict=True)]
|
|
71
|
+
await self._guard(self.store.upsert(embedded), stage="store")
|
|
72
|
+
stale = sorted(set(stored) - {chunk.id for chunk in chunks})
|
|
73
|
+
if stale:
|
|
74
|
+
await self._guard(self.store.delete_chunks(tenant_id, stale), stage="store")
|
|
75
|
+
return IndexResult(document_id=document.id, chunks=len(chunks), embedded=len(changed), deleted=len(stale))
|
|
76
|
+
|
|
77
|
+
async def remove(self, tenant_id: str, document_id: str) -> int:
|
|
78
|
+
return await self._guard(self.store.delete_document(tenant_id, document_id), stage="store")
|
|
79
|
+
|
|
80
|
+
async def search(
|
|
81
|
+
self,
|
|
82
|
+
tenant_id: str,
|
|
83
|
+
query: str,
|
|
84
|
+
*,
|
|
85
|
+
limit: int | None = None,
|
|
86
|
+
filters: Filters | None = None,
|
|
87
|
+
min_score: float | None = None,
|
|
88
|
+
) -> list[SearchHit]:
|
|
89
|
+
"""Best chunks for the query, best first, dropping those under `min_score`."""
|
|
90
|
+
if not query.strip():
|
|
91
|
+
return []
|
|
92
|
+
threshold = self.min_score if min_score is None else min_score
|
|
93
|
+
(vector,) = await self.embedder.embed([query])
|
|
94
|
+
hits = await self._guard(
|
|
95
|
+
self.store.search(tenant_id, vector, limit=limit or self.limit, filters=filters), stage="store"
|
|
96
|
+
)
|
|
97
|
+
return [hit for hit in hits if hit.score >= threshold]
|
|
98
|
+
|
|
99
|
+
@staticmethod
|
|
100
|
+
async def _guard(awaitable, *, stage: str):
|
|
101
|
+
try:
|
|
102
|
+
return await awaitable
|
|
103
|
+
except RetrievalError:
|
|
104
|
+
raise
|
|
105
|
+
except Exception as exc: # connection strings and credentials never leave as text
|
|
106
|
+
raise RetrievalError(type(exc).__name__, stage=stage) from None
|