@damiangilgonzalez/copypasta 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +227 -2
- package/catalog.yaml +140 -0
- package/cli/copypasta.js +4 -0
- package/cli/lib/catalog.js +131 -0
- package/cli/lib/commands.js +162 -0
- package/cli/lib/index.js +4 -0
- package/cli/lib/main.js +130 -0
- package/package.json +38 -4
- package/recipes/agents/agent-memory/RECIPE.md +75 -0
- package/recipes/agents/agent-memory/files/agentic/memory/__init__.py +25 -0
- package/recipes/agents/agent-memory/files/agentic/memory/base.py +101 -0
- package/recipes/agents/agent-memory/files/agentic/memory/factory.py +33 -0
- package/recipes/agents/agent-memory/files/agentic/memory/in_memory.py +59 -0
- package/recipes/agents/agent-memory/files/agentic/memory/models.py +66 -0
- package/recipes/agents/agent-memory/files/agentic/memory/settings.py +24 -0
- package/recipes/agents/agent-memory/files/agentic/memory/sql.py +177 -0
- package/recipes/agents/agent-memory/files/pytest.ini +5 -0
- package/recipes/agents/agent-memory/files/tests/__init__.py +0 -0
- package/recipes/agents/agent-memory/files/tests/memory_conformance.py +158 -0
- package/recipes/agents/agent-memory/files/tests/test_memory.py +74 -0
- package/recipes/agents/langchain-agent/RECIPE.md +101 -0
- package/recipes/agents/langchain-agent/files/agentic/__init__.py +10 -0
- package/recipes/agents/langchain-agent/files/agentic/agents/__init__.py +13 -0
- package/recipes/agents/langchain-agent/files/agentic/agents/agents.py +43 -0
- package/recipes/agents/langchain-agent/files/agentic/agents/base_agent.py +126 -0
- package/recipes/agents/langchain-agent/files/agentic/agents/models.py +46 -0
- package/recipes/agents/langchain-agent/files/agentic/context.py +21 -0
- package/recipes/agents/langchain-agent/files/agentic/llm/__init__.py +5 -0
- package/recipes/agents/langchain-agent/files/agentic/llm/models.py +45 -0
- package/recipes/agents/langchain-agent/files/agentic/llm/provider.py +78 -0
- package/recipes/agents/langchain-agent/files/agentic/llm/settings.py +62 -0
- package/recipes/agents/langchain-agent/files/agentic/middleware/__init__.py +3 -0
- package/recipes/agents/langchain-agent/files/agentic/middleware/language.py +28 -0
- package/recipes/agents/langchain-agent/files/agentic/prompts/__init__.py +3 -0
- package/recipes/agents/langchain-agent/files/agentic/prompts/assistant.en.yaml +15 -0
- package/recipes/agents/langchain-agent/files/agentic/prompts/assistant.es.yaml +15 -0
- package/recipes/agents/langchain-agent/files/agentic/prompts/loader.py +69 -0
- package/recipes/agents/langchain-agent/files/agentic/tools/__init__.py +25 -0
- package/recipes/agents/langchain-agent/files/agentic/tools/base_tool.py +62 -0
- package/recipes/agents/langchain-agent/files/agentic/tools/search_documents.py +42 -0
- package/recipes/agents/langchain-agent/files/pytest.ini +5 -0
- package/recipes/agents/langchain-agent/files/tests/__init__.py +0 -0
- package/recipes/agents/langchain-agent/files/tests/fakes.py +30 -0
- package/recipes/agents/langchain-agent/files/tests/test_agents.py +151 -0
- package/recipes/agents/langgraph-graph/RECIPE.md +63 -0
- package/recipes/agents/langgraph-graph/files/agentic/conversation.py +73 -0
- package/recipes/agents/langgraph-graph/files/agentic/graph/__init__.py +17 -0
- package/recipes/agents/langgraph-graph/files/agentic/graph/builder.py +37 -0
- package/recipes/agents/langgraph-graph/files/agentic/graph/edges.py +8 -0
- package/recipes/agents/langgraph-graph/files/agentic/graph/event_processor.py +48 -0
- package/recipes/agents/langgraph-graph/files/agentic/graph/events.py +30 -0
- package/recipes/agents/langgraph-graph/files/agentic/graph/graph.py +60 -0
- package/recipes/agents/langgraph-graph/files/agentic/graph/nodes.py +63 -0
- package/recipes/agents/langgraph-graph/files/agentic/graph/state.py +31 -0
- package/recipes/agents/langgraph-graph/files/agentic/registry.py +85 -0
- package/recipes/agents/langgraph-graph/files/tests/test_graph.py +145 -0
- package/recipes/backend/.gitkeep +0 -0
- package/recipes/backend/agent-chat-api/RECIPE.md +69 -0
- package/recipes/backend/agent-chat-api/files/api/agent/guards.py +35 -0
- package/recipes/backend/agent-chat-api/files/api/agent/router.py +86 -0
- package/recipes/backend/agent-chat-api/files/api/agent/sanitize.py +22 -0
- package/recipes/backend/agent-chat-api/files/api/agent/schemas.py +26 -0
- package/recipes/backend/agent-chat-api/files/api/agent/settings.py +14 -0
- package/recipes/backend/agent-chat-api/files/api/agent/tracing.py +38 -0
- package/recipes/backend/agent-chat-api/files/api/app.py +67 -0
- package/recipes/backend/agent-chat-api/files/api/auth.py +73 -0
- package/recipes/backend/agent-chat-api/files/pytest.ini +5 -0
- package/recipes/backend/agent-chat-api/files/tests/test_agent_api.py +237 -0
- package/recipes/backend/agent-feedback/RECIPE.md +63 -0
- package/recipes/backend/agent-feedback/files/api/feedback/__init__.py +0 -0
- package/recipes/backend/agent-feedback/files/api/feedback/router.py +72 -0
- package/recipes/backend/agent-feedback/files/api/feedback/schemas.py +44 -0
- package/recipes/backend/agent-feedback/files/feedback/__init__.py +25 -0
- package/recipes/backend/agent-feedback/files/feedback/base.py +53 -0
- package/recipes/backend/agent-feedback/files/feedback/factory.py +30 -0
- package/recipes/backend/agent-feedback/files/feedback/in_memory.py +29 -0
- package/recipes/backend/agent-feedback/files/feedback/models.py +48 -0
- package/recipes/backend/agent-feedback/files/feedback/settings.py +17 -0
- package/recipes/backend/agent-feedback/files/feedback/sql.py +102 -0
- package/recipes/backend/agent-feedback/files/pytest.ini +5 -0
- package/recipes/backend/agent-feedback/files/tests/feedback_conformance.py +89 -0
- package/recipes/backend/agent-feedback/files/tests/test_feedback.py +63 -0
- package/recipes/backend/agent-feedback/files/tests/test_feedback_api.py +95 -0
- package/recipes/backend/auth-verifier/RECIPE.md +63 -0
- package/recipes/backend/auth-verifier/files/api/auth.py +73 -0
- package/recipes/backend/auth-verifier/files/auth/__init__.py +23 -0
- package/recipes/backend/auth-verifier/files/auth/factory.py +28 -0
- package/recipes/backend/auth-verifier/files/auth/jwt_verifier.py +101 -0
- package/recipes/backend/auth-verifier/files/auth/models.py +29 -0
- package/recipes/backend/auth-verifier/files/auth/settings.py +45 -0
- package/recipes/backend/auth-verifier/files/auth/supabase.py +80 -0
- package/recipes/backend/auth-verifier/files/pytest.ini +5 -0
- package/recipes/backend/auth-verifier/files/tests/__init__.py +0 -0
- package/recipes/backend/auth-verifier/files/tests/test_auth.py +229 -0
- package/recipes/backend/conversation-history/RECIPE.md +49 -0
- package/recipes/backend/conversation-history/files/api/conversations/__init__.py +0 -0
- package/recipes/backend/conversation-history/files/api/conversations/router.py +55 -0
- package/recipes/backend/conversation-history/files/api/conversations/schemas.py +57 -0
- package/recipes/backend/conversation-history/files/pytest.ini +5 -0
- package/recipes/backend/conversation-history/files/tests/test_conversations_api.py +127 -0
- package/recipes/backend/fastapi-base/RECIPE.md +57 -0
- package/recipes/backend/fastapi-base/files/api/app.py +37 -0
- package/recipes/backend/fastapi-base/files/api/base.py +54 -0
- package/recipes/backend/fastapi-base/files/api/errors.py +31 -0
- package/recipes/backend/fastapi-base/files/api/health.py +19 -0
- package/recipes/backend/fastapi-base/files/api/logging_setup.py +26 -0
- package/recipes/backend/fastapi-base/files/api/request_id.py +62 -0
- package/recipes/backend/fastapi-base/files/api/settings.py +32 -0
- package/recipes/backend/fastapi-base/files/pytest.ini +5 -0
- package/recipes/backend/fastapi-base/files/tests/__init__.py +0 -0
- package/recipes/backend/fastapi-base/files/tests/test_api_base.py +136 -0
- package/recipes/backend/observability/RECIPE.md +68 -0
- package/recipes/backend/observability/files/observability/__init__.py +34 -0
- package/recipes/backend/observability/files/observability/factory.py +73 -0
- package/recipes/backend/observability/files/observability/langfuse_tracer.py +29 -0
- package/recipes/backend/observability/files/observability/langsmith_tracer.py +23 -0
- package/recipes/backend/observability/files/observability/models.py +90 -0
- package/recipes/backend/observability/files/observability/opik_tracer.py +30 -0
- package/recipes/backend/observability/files/observability/settings.py +58 -0
- package/recipes/backend/observability/files/observability/tracer.py +142 -0
- package/recipes/backend/observability/files/observability/usage.py +61 -0
- package/recipes/backend/observability/files/pytest.ini +5 -0
- package/recipes/backend/observability/files/tests/__init__.py +0 -0
- package/recipes/backend/observability/files/tests/test_observability.py +219 -0
- package/recipes/backend/project-settings/RECIPE.md +40 -0
- package/recipes/backend/project-settings/files/.env.example +7 -0
- package/recipes/backend/project-settings/files/core/__init__.py +0 -0
- package/recipes/backend/project-settings/files/core/config.py +41 -0
- package/recipes/backend/project-settings/files/pytest.ini +3 -0
- package/recipes/backend/project-settings/files/tests/__init__.py +0 -0
- package/recipes/backend/project-settings/files/tests/test_settings.py +44 -0
- package/recipes/backend/rag-retriever/RECIPE.md +79 -0
- package/recipes/backend/rag-retriever/files/pytest.ini +5 -0
- package/recipes/backend/rag-retriever/files/rag/__init__.py +53 -0
- package/recipes/backend/rag-retriever/files/rag/base.py +106 -0
- package/recipes/backend/rag-retriever/files/rag/chunking.py +84 -0
- package/recipes/backend/rag-retriever/files/rag/document_search.py +60 -0
- package/recipes/backend/rag-retriever/files/rag/embeddings.py +91 -0
- package/recipes/backend/rag-retriever/files/rag/factory.py +82 -0
- package/recipes/backend/rag-retriever/files/rag/in_memory.py +52 -0
- package/recipes/backend/rag-retriever/files/rag/models.py +107 -0
- package/recipes/backend/rag-retriever/files/rag/pgvector.py +138 -0
- package/recipes/backend/rag-retriever/files/rag/qdrant.py +96 -0
- package/recipes/backend/rag-retriever/files/rag/settings.py +48 -0
- package/recipes/backend/rag-retriever/files/tests/__init__.py +0 -0
- package/recipes/backend/rag-retriever/files/tests/rag_conformance.py +95 -0
- package/recipes/backend/rag-retriever/files/tests/test_rag.py +270 -0
- package/recipes/backend/rate-limit-quota/RECIPE.md +77 -0
- package/recipes/backend/rate-limit-quota/files/api/agent/guards.py +62 -0
- package/recipes/backend/rate-limit-quota/files/api/limits/__init__.py +0 -0
- package/recipes/backend/rate-limit-quota/files/api/limits/router.py +38 -0
- package/recipes/backend/rate-limit-quota/files/limits/__init__.py +43 -0
- package/recipes/backend/rate-limit-quota/files/limits/base.py +161 -0
- package/recipes/backend/rate-limit-quota/files/limits/factory.py +69 -0
- package/recipes/backend/rate-limit-quota/files/limits/in_memory.py +36 -0
- package/recipes/backend/rate-limit-quota/files/limits/models.py +108 -0
- package/recipes/backend/rate-limit-quota/files/limits/settings.py +32 -0
- package/recipes/backend/rate-limit-quota/files/limits/sql.py +94 -0
- package/recipes/backend/rate-limit-quota/files/pytest.ini +5 -0
- package/recipes/backend/rate-limit-quota/files/tests/limits_conformance.py +146 -0
- package/recipes/backend/rate-limit-quota/files/tests/test_limits.py +167 -0
- package/recipes/backend/rate-limit-quota/files/tests/test_limits_api.py +100 -0
- package/recipes/backend/voice-providers/RECIPE.md +73 -0
- package/recipes/backend/voice-providers/files/api/__init__.py +0 -0
- package/recipes/backend/voice-providers/files/api/voice/__init__.py +0 -0
- package/recipes/backend/voice-providers/files/api/voice/router.py +113 -0
- package/recipes/backend/voice-providers/files/pytest.ini +5 -0
- package/recipes/backend/voice-providers/files/tests/__init__.py +0 -0
- package/recipes/backend/voice-providers/files/tests/fake_openai.py +44 -0
- package/recipes/backend/voice-providers/files/tests/test_voice.py +164 -0
- package/recipes/backend/voice-providers/files/tests/test_voice_api.py +106 -0
- package/recipes/backend/voice-providers/files/tests/voice_conformance.py +80 -0
- package/recipes/backend/voice-providers/files/voice/__init__.py +29 -0
- package/recipes/backend/voice-providers/files/voice/base.py +116 -0
- package/recipes/backend/voice-providers/files/voice/factory.py +48 -0
- package/recipes/backend/voice-providers/files/voice/fake.py +51 -0
- package/recipes/backend/voice-providers/files/voice/models.py +85 -0
- package/recipes/backend/voice-providers/files/voice/openai_provider.py +116 -0
- package/recipes/backend/voice-providers/files/voice/settings.py +48 -0
- package/recipes/frontend/.gitkeep +0 -0
- package/recipes/frontend/playwright-e2e/RECIPE.md +48 -0
- package/recipes/frontend/playwright-e2e/files/.github/workflows/frontend.yml +35 -0
- package/recipes/frontend/playwright-e2e/files/e2e/helpers.ts +25 -0
- package/recipes/frontend/playwright-e2e/files/e2e/home.spec.ts +10 -0
- package/recipes/frontend/playwright-e2e/files/e2e/login.spec.ts +26 -0
- package/recipes/frontend/playwright-e2e/files/e2e/navigation.spec.ts +27 -0
- package/recipes/frontend/playwright-e2e/files/playwright.config.ts +35 -0
- package/recipes/frontend/react-agent-chat/RECIPE.md +69 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/chat-api.ts +27 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/chat.css +18 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/chat.test.tsx +139 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/components/ChatView.tsx +22 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/components/Composer.tsx +40 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/components/Markdown.tsx +20 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/components/MessageList.tsx +42 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/events.ts +42 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/fake-chat-api.ts +44 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/index.ts +6 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/sse-parser.test.ts +97 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/sse-parser.ts +85 -0
- package/recipes/frontend/react-agent-chat/files/src/features/chat/use-chat.ts +136 -0
- package/recipes/frontend/react-app-router/RECIPE.md +58 -0
- package/recipes/frontend/react-app-router/files/src/App.test.tsx +54 -0
- package/recipes/frontend/react-app-router/files/src/App.tsx +14 -0
- package/recipes/frontend/react-app-router/files/src/router/ErrorScreen.tsx +21 -0
- package/recipes/frontend/react-app-router/files/src/router/Layout.tsx +28 -0
- package/recipes/frontend/react-app-router/files/src/router/NotFoundScreen.tsx +12 -0
- package/recipes/frontend/react-app-router/files/src/router/index.ts +1 -0
- package/recipes/frontend/react-app-router/files/src/router/router.css +29 -0
- package/recipes/frontend/react-app-router/files/src/router/router.test.tsx +65 -0
- package/recipes/frontend/react-app-router/files/src/router/routes.tsx +59 -0
- package/recipes/frontend/react-app-router/files/src/screens/SettingsScreen.tsx +10 -0
- package/recipes/frontend/react-supabase-auth/RECIPE.md +77 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/LoginScreen.tsx +64 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/SignOutButton.tsx +12 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/auth.css +33 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/auth.test.ts +149 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/auth.ts +59 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/fake-auth.ts +87 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/fake-gotrue.ts +85 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/index.ts +7 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/make-auth.ts +20 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/session-context.tsx +35 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/session-store.ts +63 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/session.test.tsx +56 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/settings.ts +30 -0
- package/recipes/frontend/react-supabase-auth/files/src/auth/supabase-auth.ts +85 -0
- package/recipes/frontend/vite-react-base/RECIPE.md +69 -0
- package/recipes/frontend/vite-react-base/files/.env +6 -0
- package/recipes/frontend/vite-react-base/files/.env.fake +2 -0
- package/recipes/frontend/vite-react-base/files/index.html +12 -0
- package/recipes/frontend/vite-react-base/files/package.json +30 -0
- package/recipes/frontend/vite-react-base/files/src/App.test.tsx +49 -0
- package/recipes/frontend/vite-react-base/files/src/App.tsx +13 -0
- package/recipes/frontend/vite-react-base/files/src/api/api.ts +37 -0
- package/recipes/frontend/vite-react-base/files/src/api/fake/fake-api.ts +29 -0
- package/recipes/frontend/vite-react-base/files/src/api/http-api.ts +16 -0
- package/recipes/frontend/vite-react-base/files/src/api/http-client.test.ts +84 -0
- package/recipes/frontend/vite-react-base/files/src/api/http-client.ts +94 -0
- package/recipes/frontend/vite-react-base/files/src/api/make-api.ts +30 -0
- package/recipes/frontend/vite-react-base/files/src/api/types.ts +7 -0
- package/recipes/frontend/vite-react-base/files/src/app-context.tsx +24 -0
- package/recipes/frontend/vite-react-base/files/src/auth/token-provider.ts +27 -0
- package/recipes/frontend/vite-react-base/files/src/config.ts +18 -0
- package/recipes/frontend/vite-react-base/files/src/hooks/use-load.ts +46 -0
- package/recipes/frontend/vite-react-base/files/src/main.tsx +27 -0
- package/recipes/frontend/vite-react-base/files/src/screens/HomeScreen.tsx +28 -0
- package/recipes/frontend/vite-react-base/files/src/styles/global.css +25 -0
- package/recipes/frontend/vite-react-base/files/src/styles/tokens.css +33 -0
- package/recipes/frontend/vite-react-base/files/src/styles/tokens.test.ts +45 -0
- package/recipes/frontend/vite-react-base/files/src/test-setup.ts +8 -0
- package/recipes/frontend/vite-react-base/files/src/vite-env.d.ts +11 -0
- package/recipes/frontend/vite-react-base/files/tsconfig.json +18 -0
- package/recipes/frontend/vite-react-base/files/vite.config.ts +19 -0
- package/unpacked-files.json +3 -0
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
"""Split a document into chunks of about `size` characters with `overlap` characters shared
|
|
2
|
+
between neighbours. Recursive: paragraphs first, then lines, sentences, words, characters
|
|
3
|
+
(the RecursiveCharacterTextSplitter idea, without the dependency).
|
|
4
|
+
|
|
5
|
+
Characters, not tokens: close enough for `text-embedding-3-*` (~4 chars per token) and no
|
|
6
|
+
tokenizer to ship. If you need token-exact windows, replace `TextChunker.split`.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from pydantic import BaseModel, ConfigDict, Field, model_validator
|
|
10
|
+
|
|
11
|
+
from .models import Chunk, Document
|
|
12
|
+
|
|
13
|
+
SEPARATORS = ["\n\n", "\n", ". ", " ", ""]
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class TextChunker(BaseModel):
|
|
17
|
+
model_config = ConfigDict(frozen=True)
|
|
18
|
+
|
|
19
|
+
size: int = Field(default=1000, gt=0)
|
|
20
|
+
overlap: int = Field(default=200, ge=0)
|
|
21
|
+
|
|
22
|
+
@model_validator(mode="after")
|
|
23
|
+
def _overlap_smaller_than_size(self) -> "TextChunker":
|
|
24
|
+
if self.overlap >= self.size:
|
|
25
|
+
raise ValueError("chunk overlap must be smaller than chunk size")
|
|
26
|
+
return self
|
|
27
|
+
|
|
28
|
+
def split(self, text: str) -> list[str]:
|
|
29
|
+
text = text.strip()
|
|
30
|
+
if not text:
|
|
31
|
+
return []
|
|
32
|
+
pieces = self._split(text, SEPARATORS)
|
|
33
|
+
return [p for p in self._merge(pieces) if p.strip()]
|
|
34
|
+
|
|
35
|
+
def chunk(self, tenant_id: str, document: Document) -> list[Chunk]:
|
|
36
|
+
return [
|
|
37
|
+
Chunk(
|
|
38
|
+
id=Chunk.id_for(tenant_id, document.id, index),
|
|
39
|
+
tenant_id=tenant_id,
|
|
40
|
+
document_id=document.id,
|
|
41
|
+
chunk_index=index,
|
|
42
|
+
text=piece,
|
|
43
|
+
content_hash=Chunk.hash_of(piece),
|
|
44
|
+
title=document.title,
|
|
45
|
+
source=document.source,
|
|
46
|
+
metadata=dict(document.metadata),
|
|
47
|
+
)
|
|
48
|
+
for index, piece in enumerate(self.split(document.text))
|
|
49
|
+
]
|
|
50
|
+
|
|
51
|
+
def _split(self, text: str, separators: list[str]) -> list[str]:
|
|
52
|
+
"""Pieces no longer than `size`, cut at the coarsest separator that works."""
|
|
53
|
+
if len(text) <= self.size:
|
|
54
|
+
return [text]
|
|
55
|
+
separator, rest = separators[0], separators[1:]
|
|
56
|
+
parts = list(text) if separator == "" else [p + separator for p in text.split(separator)]
|
|
57
|
+
if separator:
|
|
58
|
+
parts[-1] = parts[-1].removesuffix(separator)
|
|
59
|
+
out: list[str] = []
|
|
60
|
+
for part in parts:
|
|
61
|
+
if not part:
|
|
62
|
+
continue
|
|
63
|
+
out.extend(self._split(part, rest) if len(part) > self.size and rest else [part])
|
|
64
|
+
return out
|
|
65
|
+
|
|
66
|
+
def _merge(self, pieces: list[str]) -> list[str]:
|
|
67
|
+
"""Greedy merge of small pieces up to `size`, carrying `overlap` chars into the next chunk."""
|
|
68
|
+
chunks: list[str] = []
|
|
69
|
+
current = ""
|
|
70
|
+
for piece in pieces:
|
|
71
|
+
if current and len(current) + len(piece) > self.size:
|
|
72
|
+
chunks.append(current.strip())
|
|
73
|
+
current = self._tail(current)
|
|
74
|
+
current += piece
|
|
75
|
+
if current.strip():
|
|
76
|
+
chunks.append(current.strip())
|
|
77
|
+
return chunks
|
|
78
|
+
|
|
79
|
+
def _tail(self, text: str) -> str:
|
|
80
|
+
"""The last `overlap` characters, started at a word boundary."""
|
|
81
|
+
if not self.overlap:
|
|
82
|
+
return ""
|
|
83
|
+
tail = text[-self.overlap :]
|
|
84
|
+
return tail[tail.index(" ") + 1 :] if " " in tail else tail
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
"""The service the agent's `search_documents` tool expects (recipe `langchain-agent`,
|
|
2
|
+
`DocumentSearch` protocol): `search(tenant_id, query, limit) -> list[{"title", "snippet", ...}]`.
|
|
3
|
+
|
|
4
|
+
Also `format_hits`: the context block for a prompt, numbered, with the citation and WITHOUT scores
|
|
5
|
+
or tenant (the LLM reasons on text; scores travel on another channel if the UI needs them).
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from pydantic import BaseModel, ConfigDict
|
|
9
|
+
|
|
10
|
+
from .base import Retriever
|
|
11
|
+
from .models import Filters, SearchHit
|
|
12
|
+
|
|
13
|
+
SNIPPET_LENGTH = 300
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class DocumentSearchResult(BaseModel):
|
|
17
|
+
model_config = ConfigDict(frozen=True)
|
|
18
|
+
|
|
19
|
+
title: str
|
|
20
|
+
snippet: str
|
|
21
|
+
source: str | None
|
|
22
|
+
document_id: str
|
|
23
|
+
score: float
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class DocumentSearchService:
|
|
27
|
+
def __init__(self, retriever: Retriever, *, filters: Filters | None = None) -> None:
|
|
28
|
+
self._retriever = retriever
|
|
29
|
+
self._filters = filters
|
|
30
|
+
|
|
31
|
+
async def search(self, tenant_id: str, query: str, limit: int) -> list[dict]:
|
|
32
|
+
hits = await self._retriever.search(tenant_id, query, limit=limit, filters=self._filters)
|
|
33
|
+
return [
|
|
34
|
+
DocumentSearchResult(
|
|
35
|
+
title=hit.chunk.title or hit.chunk.document_id,
|
|
36
|
+
snippet=snippet(hit.chunk.text),
|
|
37
|
+
source=hit.chunk.source,
|
|
38
|
+
document_id=hit.chunk.document_id,
|
|
39
|
+
score=hit.score,
|
|
40
|
+
).model_dump()
|
|
41
|
+
for hit in hits
|
|
42
|
+
]
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def snippet(text: str, length: int = SNIPPET_LENGTH) -> str:
|
|
46
|
+
text = " ".join(text.split())
|
|
47
|
+
return text if len(text) <= length else text[: length - 1].rstrip() + "…"
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def format_hits(hits: list[SearchHit], *, empty: str = "No relevant documents.") -> str:
|
|
51
|
+
"""`[1] Title (source)\\n<text>` per hit, separated by blank lines."""
|
|
52
|
+
if not hits:
|
|
53
|
+
return empty
|
|
54
|
+
blocks = []
|
|
55
|
+
for number, hit in enumerate(hits, start=1):
|
|
56
|
+
head = hit.chunk.title or hit.chunk.document_id
|
|
57
|
+
if hit.chunk.source:
|
|
58
|
+
head += f" ({hit.chunk.source})"
|
|
59
|
+
blocks.append(f"[{number}] {head}\n{hit.chunk.text.strip()}")
|
|
60
|
+
return "\n\n".join(blocks)
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"""Embedders: `Embedder` is the contract, `OpenAIEmbedder` the real one, `HashingEmbedder` the
|
|
2
|
+
local one (deterministic bag of words, no network) for development and tests.
|
|
3
|
+
|
|
4
|
+
Every embedder states its `model` and `dimension`: the store's collection is created with that
|
|
5
|
+
dimension and the chunk hash is only valid for that model, so changing either means re-indexing.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import hashlib
|
|
9
|
+
import math
|
|
10
|
+
import re
|
|
11
|
+
from abc import ABC, abstractmethod
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
from pydantic import SecretStr
|
|
15
|
+
|
|
16
|
+
from .models import RetrievalError
|
|
17
|
+
|
|
18
|
+
TOKEN = re.compile(r"\w+", re.UNICODE)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class Embedder(ABC):
|
|
22
|
+
def __init__(self, *, model: str, dimension: int) -> None:
|
|
23
|
+
self.model = model
|
|
24
|
+
self.dimension = dimension
|
|
25
|
+
|
|
26
|
+
async def embed(self, texts: list[str]) -> list[list[float]]:
|
|
27
|
+
"""One vector per text, same order. Any failure -> `RetrievalError(kind, stage="embed")`."""
|
|
28
|
+
if not texts:
|
|
29
|
+
return []
|
|
30
|
+
try:
|
|
31
|
+
vectors = await self._embed(texts)
|
|
32
|
+
except RetrievalError:
|
|
33
|
+
raise
|
|
34
|
+
except Exception as exc: # the message may hold the API key: keep the class name only
|
|
35
|
+
raise RetrievalError(type(exc).__name__, stage="embed") from None
|
|
36
|
+
if len(vectors) != len(texts):
|
|
37
|
+
raise RetrievalError("MissingVectors", stage="embed")
|
|
38
|
+
return vectors
|
|
39
|
+
|
|
40
|
+
@abstractmethod
|
|
41
|
+
async def _embed(self, texts: list[str]) -> list[list[float]]: ...
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class HashingEmbedder(Embedder):
|
|
45
|
+
"""Hashing trick: each word lands in a bucket, counts are L2-normalised. Texts sharing words
|
|
46
|
+
get a high cosine, unrelated texts a low one: good enough for tests and local demos."""
|
|
47
|
+
|
|
48
|
+
def __init__(self, *, dimension: int = 256) -> None:
|
|
49
|
+
super().__init__(model="hashing", dimension=dimension)
|
|
50
|
+
|
|
51
|
+
async def _embed(self, texts: list[str]) -> list[list[float]]:
|
|
52
|
+
return [self._vector(text) for text in texts]
|
|
53
|
+
|
|
54
|
+
def _vector(self, text: str) -> list[float]:
|
|
55
|
+
vector = [0.0] * self.dimension
|
|
56
|
+
for token in TOKEN.findall(text.lower()):
|
|
57
|
+
digest = hashlib.sha256(token.encode("utf-8")).digest()
|
|
58
|
+
vector[int.from_bytes(digest[:4], "big") % self.dimension] += 1.0
|
|
59
|
+
norm = math.sqrt(sum(v * v for v in vector))
|
|
60
|
+
return [v / norm for v in vector] if norm else vector
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
class OpenAIEmbedder(Embedder):
|
|
64
|
+
"""`text-embedding-3-*` through the official SDK; `dimensions` is sent so the vectors match
|
|
65
|
+
the collection. Batches of `batch_size` texts, output re-ordered by `index`."""
|
|
66
|
+
|
|
67
|
+
def __init__(
|
|
68
|
+
self,
|
|
69
|
+
*,
|
|
70
|
+
api_key: SecretStr,
|
|
71
|
+
model: str = "text-embedding-3-small",
|
|
72
|
+
dimension: int = 1536,
|
|
73
|
+
batch_size: int = 100,
|
|
74
|
+
timeout: float = 30.0,
|
|
75
|
+
client: Any | None = None,
|
|
76
|
+
) -> None:
|
|
77
|
+
super().__init__(model=model, dimension=dimension)
|
|
78
|
+
self.batch_size = batch_size
|
|
79
|
+
if client is None:
|
|
80
|
+
from openai import AsyncOpenAI # lazy: install `openai` only if you use this provider
|
|
81
|
+
|
|
82
|
+
client = AsyncOpenAI(api_key=api_key.get_secret_value(), timeout=timeout)
|
|
83
|
+
self._client = client
|
|
84
|
+
|
|
85
|
+
async def _embed(self, texts: list[str]) -> list[list[float]]:
|
|
86
|
+
vectors: list[list[float]] = []
|
|
87
|
+
for start in range(0, len(texts), self.batch_size):
|
|
88
|
+
batch = texts[start : start + self.batch_size]
|
|
89
|
+
response = await self._client.embeddings.create(model=self.model, input=batch, dimensions=self.dimension)
|
|
90
|
+
vectors.extend(item.embedding for item in sorted(response.data, key=lambda item: item.index))
|
|
91
|
+
return vectors
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""The one place that builds the embedder, the store and the `Retriever` from settings."""
|
|
2
|
+
|
|
3
|
+
from collections.abc import Callable
|
|
4
|
+
from typing import Any
|
|
5
|
+
|
|
6
|
+
from .base import Retriever, VectorStore
|
|
7
|
+
from .chunking import TextChunker
|
|
8
|
+
from .embeddings import Embedder, HashingEmbedder, OpenAIEmbedder
|
|
9
|
+
from .models import EmbeddingsProvider, RagBackend
|
|
10
|
+
from .settings import RagSettings
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
class EmbedderFactory:
|
|
14
|
+
@staticmethod
|
|
15
|
+
def create(settings: RagSettings) -> Embedder:
|
|
16
|
+
match settings.embeddings_provider:
|
|
17
|
+
case EmbeddingsProvider.HASHING:
|
|
18
|
+
return HashingEmbedder(dimension=settings.embedding_dimension)
|
|
19
|
+
case EmbeddingsProvider.OPENAI:
|
|
20
|
+
assert settings.openai_api_key is not None # the settings validator guarantees it
|
|
21
|
+
return OpenAIEmbedder(
|
|
22
|
+
api_key=settings.openai_api_key,
|
|
23
|
+
model=settings.embedding_model,
|
|
24
|
+
dimension=settings.embedding_dimension,
|
|
25
|
+
)
|
|
26
|
+
raise ValueError(f"Unknown embeddings provider: {settings.embeddings_provider!r}")
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class VectorStoreFactory:
|
|
30
|
+
@staticmethod
|
|
31
|
+
def create(settings: RagSettings, *, session_factory: Callable[[], Any] | None = None) -> VectorStore:
|
|
32
|
+
dimension = settings.embedding_dimension
|
|
33
|
+
match settings.rag_backend:
|
|
34
|
+
case RagBackend.MEMORY:
|
|
35
|
+
from .in_memory import InMemoryVectorStore
|
|
36
|
+
|
|
37
|
+
return InMemoryVectorStore()
|
|
38
|
+
case RagBackend.QDRANT:
|
|
39
|
+
from qdrant_client import AsyncQdrantClient
|
|
40
|
+
|
|
41
|
+
from .qdrant import QdrantVectorStore
|
|
42
|
+
|
|
43
|
+
key = settings.qdrant_api_key.get_secret_value() if settings.qdrant_api_key else None
|
|
44
|
+
client = AsyncQdrantClient(url=settings.qdrant_url, api_key=key)
|
|
45
|
+
return QdrantVectorStore(client, collection=settings.qdrant_collection, dimension=dimension)
|
|
46
|
+
case RagBackend.PGVECTOR:
|
|
47
|
+
if session_factory is None:
|
|
48
|
+
raise ValueError("RAG_BACKEND=pgvector needs the app's session_factory")
|
|
49
|
+
from .pgvector import PgVectorStore
|
|
50
|
+
|
|
51
|
+
return PgVectorStore(session_factory, dimension=dimension)
|
|
52
|
+
raise ValueError(f"Unknown RAG backend: {settings.rag_backend!r}")
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class RagFactory:
|
|
56
|
+
@staticmethod
|
|
57
|
+
def create(
|
|
58
|
+
settings: RagSettings,
|
|
59
|
+
*,
|
|
60
|
+
session_factory: Callable[[], Any] | None = None,
|
|
61
|
+
embedder: Embedder | None = None,
|
|
62
|
+
store: VectorStore | None = None,
|
|
63
|
+
) -> Retriever:
|
|
64
|
+
"""Call it once at startup (FastAPI `lifespan`), then `await retriever.store.ensure_ready()`:
|
|
65
|
+
|
|
66
|
+
retriever = RagFactory.create(get_settings(), session_factory=SessionLocal)
|
|
67
|
+
|
|
68
|
+
`embedder=` / `store=` replace the one the settings would build (tests, custom backends).
|
|
69
|
+
"""
|
|
70
|
+
embedder = embedder or EmbedderFactory.create(settings)
|
|
71
|
+
store = store or VectorStoreFactory.create(settings, session_factory=session_factory)
|
|
72
|
+
if embedder.dimension != settings.embedding_dimension:
|
|
73
|
+
raise ValueError(
|
|
74
|
+
f"embedder dimension {embedder.dimension} != EMBEDDING_DIMENSION {settings.embedding_dimension}"
|
|
75
|
+
)
|
|
76
|
+
return Retriever(
|
|
77
|
+
embedder=embedder,
|
|
78
|
+
store=store,
|
|
79
|
+
chunker=TextChunker(size=settings.rag_chunk_size, overlap=settings.rag_chunk_overlap),
|
|
80
|
+
limit=settings.rag_search_limit,
|
|
81
|
+
min_score=settings.rag_min_score,
|
|
82
|
+
)
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
"""Vector store in RAM: one process, lost on restart. Development and tests, and the reference
|
|
2
|
+
every other backend must match (conformance tests)."""
|
|
3
|
+
|
|
4
|
+
import math
|
|
5
|
+
|
|
6
|
+
from .base import VectorStore
|
|
7
|
+
from .models import EmbeddedChunk, Filters, SearchHit
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def cosine(a: list[float], b: list[float]) -> float:
|
|
11
|
+
dot = sum(x * y for x, y in zip(a, b, strict=True))
|
|
12
|
+
norm = math.sqrt(sum(x * x for x in a)) * math.sqrt(sum(y * y for y in b))
|
|
13
|
+
return dot / norm if norm else 0.0
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class InMemoryVectorStore(VectorStore):
|
|
17
|
+
def __init__(self) -> None:
|
|
18
|
+
self._rows: dict[str, EmbeddedChunk] = {}
|
|
19
|
+
|
|
20
|
+
async def upsert(self, chunks: list[EmbeddedChunk]) -> None:
|
|
21
|
+
for item in chunks:
|
|
22
|
+
self._rows[item.chunk.id] = item
|
|
23
|
+
|
|
24
|
+
async def existing_hashes(self, tenant_id: str, document_id: str) -> dict[str, str]:
|
|
25
|
+
return {
|
|
26
|
+
row.chunk.id: row.chunk.content_hash
|
|
27
|
+
for row in self._rows.values()
|
|
28
|
+
if row.chunk.tenant_id == tenant_id and row.chunk.document_id == document_id
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
async def delete_chunks(self, tenant_id: str, chunk_ids: list[str]) -> None:
|
|
32
|
+
for chunk_id in chunk_ids:
|
|
33
|
+
row = self._rows.get(chunk_id)
|
|
34
|
+
if row is not None and row.chunk.tenant_id == tenant_id:
|
|
35
|
+
del self._rows[chunk_id]
|
|
36
|
+
|
|
37
|
+
async def delete_document(self, tenant_id: str, document_id: str) -> int:
|
|
38
|
+
ids = list(await self.existing_hashes(tenant_id, document_id))
|
|
39
|
+
await self.delete_chunks(tenant_id, ids)
|
|
40
|
+
return len(ids)
|
|
41
|
+
|
|
42
|
+
async def search(
|
|
43
|
+
self, tenant_id: str, vector: list[float], *, limit: int, filters: Filters | None = None
|
|
44
|
+
) -> list[SearchHit]:
|
|
45
|
+
wanted = filters or {}
|
|
46
|
+
hits = [
|
|
47
|
+
SearchHit(chunk=row.chunk, score=max(0.0, min(1.0, cosine(vector, row.vector))))
|
|
48
|
+
for row in self._rows.values()
|
|
49
|
+
if row.chunk.tenant_id == tenant_id and all(row.chunk.metadata.get(k) == v for k, v in wanted.items())
|
|
50
|
+
]
|
|
51
|
+
hits.sort(key=lambda hit: (-hit.score, hit.chunk.document_id, hit.chunk.chunk_index))
|
|
52
|
+
return hits[:limit]
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"""Data of the retrieval layer. Everything is Pydantic; the pieces the stores persist are frozen.
|
|
2
|
+
|
|
3
|
+
A `Document` is what the app ingests (one PDF, one page, one record). The chunker turns it into
|
|
4
|
+
`Chunk`s with a deterministic id (`uuid5(tenant:document:index)`) and a `content_hash`, so
|
|
5
|
+
indexing the same document twice is a no-op and changing one paragraph re-embeds one chunk.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import hashlib
|
|
9
|
+
import uuid
|
|
10
|
+
from enum import StrEnum
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
from pydantic import BaseModel, ConfigDict, Field
|
|
14
|
+
|
|
15
|
+
CHUNK_NAMESPACE = uuid.UUID("6f1c9b7e-2d47-4c8b-9e1a-3b5c7d9e1f20")
|
|
16
|
+
|
|
17
|
+
Scalar = str | int | float | bool
|
|
18
|
+
Filters = dict[str, Scalar]
|
|
19
|
+
"""Equality filters on `Chunk.metadata` keys (`{"category": "manual"}`). Tenant is never a filter:
|
|
20
|
+
it is a mandatory argument everywhere."""
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class RagBackend(StrEnum):
|
|
24
|
+
MEMORY = "memory"
|
|
25
|
+
QDRANT = "qdrant"
|
|
26
|
+
PGVECTOR = "pgvector"
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class EmbeddingsProvider(StrEnum):
|
|
30
|
+
OPENAI = "openai"
|
|
31
|
+
HASHING = "hashing" # local, deterministic, no network: dev and tests
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class Document(BaseModel):
|
|
35
|
+
"""What the app hands to `Retriever.index`. `id` must be stable across re-ingestions
|
|
36
|
+
(a path, a primary key), never random: it is what makes indexing idempotent."""
|
|
37
|
+
|
|
38
|
+
model_config = ConfigDict(frozen=True)
|
|
39
|
+
|
|
40
|
+
id: str = Field(min_length=1)
|
|
41
|
+
text: str
|
|
42
|
+
title: str = ""
|
|
43
|
+
source: str | None = Field(default=None, description="URL or path shown to the user as the citation.")
|
|
44
|
+
metadata: dict[str, Scalar] = Field(default_factory=dict, description="Filterable fields (category, year...).")
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class Chunk(BaseModel):
|
|
48
|
+
model_config = ConfigDict(frozen=True)
|
|
49
|
+
|
|
50
|
+
id: str
|
|
51
|
+
tenant_id: str
|
|
52
|
+
document_id: str
|
|
53
|
+
chunk_index: int = Field(ge=0)
|
|
54
|
+
text: str
|
|
55
|
+
content_hash: str
|
|
56
|
+
title: str = ""
|
|
57
|
+
source: str | None = None
|
|
58
|
+
metadata: dict[str, Scalar] = Field(default_factory=dict)
|
|
59
|
+
|
|
60
|
+
@staticmethod
|
|
61
|
+
def id_for(tenant_id: str, document_id: str, chunk_index: int) -> str:
|
|
62
|
+
return str(uuid.uuid5(CHUNK_NAMESPACE, f"{tenant_id}:{document_id}:{chunk_index}"))
|
|
63
|
+
|
|
64
|
+
@staticmethod
|
|
65
|
+
def hash_of(text: str) -> str:
|
|
66
|
+
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class EmbeddedChunk(BaseModel):
|
|
70
|
+
model_config = ConfigDict(frozen=True)
|
|
71
|
+
|
|
72
|
+
chunk: Chunk
|
|
73
|
+
vector: list[float]
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
class SearchHit(BaseModel):
|
|
77
|
+
model_config = ConfigDict(frozen=True)
|
|
78
|
+
|
|
79
|
+
chunk: Chunk
|
|
80
|
+
score: float = Field(description="Cosine similarity in [0, 1]: 1 = identical direction.")
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class IndexResult(BaseModel):
|
|
84
|
+
model_config = ConfigDict(frozen=True)
|
|
85
|
+
|
|
86
|
+
document_id: str
|
|
87
|
+
chunks: int
|
|
88
|
+
embedded: int = Field(description="Chunks that were new or changed and went through the embedder.")
|
|
89
|
+
deleted: int = Field(description="Stale chunks removed because the document got shorter.")
|
|
90
|
+
|
|
91
|
+
@property
|
|
92
|
+
def unchanged(self) -> int:
|
|
93
|
+
return self.chunks - self.embedded
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
class RetrievalError(Exception):
|
|
97
|
+
"""The only error the retrieval path raises. `kind` is the class name of the underlying error,
|
|
98
|
+
never its text (it may carry a connection string or an API key)."""
|
|
99
|
+
|
|
100
|
+
def __init__(self, kind: str, *, stage: str) -> None:
|
|
101
|
+
super().__init__(f"{stage}: {kind}")
|
|
102
|
+
self.kind = kind
|
|
103
|
+
self.stage = stage
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def metadata_of(payload: dict[str, Any]) -> dict[str, Scalar]:
|
|
107
|
+
return {k: v for k, v in payload.items() if isinstance(v, str | int | float | bool)}
|
|
@@ -0,0 +1,138 @@
|
|
|
1
|
+
"""Postgres + pgvector backend over SQLAlchemy Core (`text()` SQL, synchronous session in
|
|
2
|
+
`asyncio.to_thread`, like the other SQL recipes). One table `rag_chunks`, cosine `<=>`.
|
|
3
|
+
|
|
4
|
+
Needs `CREATE EXTENSION vector` and the table from `schema_sql(dimension)` (run it in your
|
|
5
|
+
migration). Only tested against a real Postgres: see `RAG_TEST_DATABASE_URL` in the tests.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import asyncio
|
|
9
|
+
import json
|
|
10
|
+
from collections.abc import Callable
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
from sqlalchemy import bindparam, text
|
|
14
|
+
|
|
15
|
+
from .base import VectorStore
|
|
16
|
+
from .models import Chunk, EmbeddedChunk, Filters, SearchHit, metadata_of
|
|
17
|
+
|
|
18
|
+
TABLE = "rag_chunks"
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def schema_sql(dimension: int) -> str:
|
|
22
|
+
return f"""
|
|
23
|
+
CREATE EXTENSION IF NOT EXISTS vector;
|
|
24
|
+
CREATE TABLE IF NOT EXISTS {TABLE} (
|
|
25
|
+
id TEXT PRIMARY KEY,
|
|
26
|
+
tenant_id TEXT NOT NULL,
|
|
27
|
+
document_id TEXT NOT NULL,
|
|
28
|
+
chunk_index INTEGER NOT NULL,
|
|
29
|
+
title TEXT NOT NULL DEFAULT '',
|
|
30
|
+
source TEXT,
|
|
31
|
+
text TEXT NOT NULL,
|
|
32
|
+
content_hash TEXT NOT NULL,
|
|
33
|
+
metadata JSONB NOT NULL DEFAULT '{{}}',
|
|
34
|
+
embedding vector({dimension}) NOT NULL
|
|
35
|
+
);
|
|
36
|
+
CREATE INDEX IF NOT EXISTS ix_{TABLE}_tenant_document ON {TABLE} (tenant_id, document_id);
|
|
37
|
+
CREATE INDEX IF NOT EXISTS ix_{TABLE}_embedding ON {TABLE} USING hnsw (embedding vector_cosine_ops);
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class PgVectorStore(VectorStore):
|
|
42
|
+
def __init__(self, session_factory: Callable[[], Any], *, dimension: int) -> None:
|
|
43
|
+
self._session_factory = session_factory
|
|
44
|
+
self.dimension = dimension
|
|
45
|
+
|
|
46
|
+
async def ensure_ready(self) -> None:
|
|
47
|
+
def run() -> None:
|
|
48
|
+
with self._session_factory() as session:
|
|
49
|
+
for statement in schema_sql(self.dimension).split(";"):
|
|
50
|
+
if statement.strip():
|
|
51
|
+
session.execute(text(statement))
|
|
52
|
+
session.commit()
|
|
53
|
+
|
|
54
|
+
await asyncio.to_thread(run)
|
|
55
|
+
|
|
56
|
+
async def upsert(self, chunks: list[EmbeddedChunk]) -> None:
|
|
57
|
+
if not chunks:
|
|
58
|
+
return
|
|
59
|
+
rows = [
|
|
60
|
+
{**c.chunk.model_dump(exclude={"metadata"}), "metadata": json.dumps(c.chunk.metadata), "embedding": _literal(c.vector)}
|
|
61
|
+
for c in chunks
|
|
62
|
+
]
|
|
63
|
+
await self._run(
|
|
64
|
+
text(f"""
|
|
65
|
+
INSERT INTO {TABLE} (id, tenant_id, document_id, chunk_index, title, source, text, content_hash, metadata, embedding)
|
|
66
|
+
VALUES (:id, :tenant_id, :document_id, :chunk_index, :title, :source, :text, :content_hash,
|
|
67
|
+
CAST(:metadata AS jsonb), CAST(:embedding AS vector))
|
|
68
|
+
ON CONFLICT (id) DO UPDATE SET
|
|
69
|
+
title = EXCLUDED.title, source = EXCLUDED.source, text = EXCLUDED.text,
|
|
70
|
+
content_hash = EXCLUDED.content_hash, metadata = EXCLUDED.metadata, embedding = EXCLUDED.embedding
|
|
71
|
+
"""),
|
|
72
|
+
rows,
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
async def existing_hashes(self, tenant_id: str, document_id: str) -> dict[str, str]:
|
|
76
|
+
rows = await self._run(
|
|
77
|
+
text(f"SELECT id, content_hash FROM {TABLE} WHERE tenant_id = :tenant_id AND document_id = :document_id"),
|
|
78
|
+
{"tenant_id": tenant_id, "document_id": document_id},
|
|
79
|
+
)
|
|
80
|
+
return {row.id: row.content_hash for row in rows}
|
|
81
|
+
|
|
82
|
+
async def delete_chunks(self, tenant_id: str, chunk_ids: list[str]) -> None:
|
|
83
|
+
if chunk_ids:
|
|
84
|
+
await self._run(
|
|
85
|
+
text(f"DELETE FROM {TABLE} WHERE tenant_id = :tenant_id AND id IN :ids").bindparams(
|
|
86
|
+
bindparam("ids", expanding=True)
|
|
87
|
+
),
|
|
88
|
+
{"tenant_id": tenant_id, "ids": chunk_ids},
|
|
89
|
+
)
|
|
90
|
+
|
|
91
|
+
async def delete_document(self, tenant_id: str, document_id: str) -> int:
|
|
92
|
+
rows = await self._run(
|
|
93
|
+
text(f"DELETE FROM {TABLE} WHERE tenant_id = :tenant_id AND document_id = :document_id RETURNING id"),
|
|
94
|
+
{"tenant_id": tenant_id, "document_id": document_id},
|
|
95
|
+
)
|
|
96
|
+
return len(rows)
|
|
97
|
+
|
|
98
|
+
async def search(
|
|
99
|
+
self, tenant_id: str, vector: list[float], *, limit: int, filters: Filters | None = None
|
|
100
|
+
) -> list[SearchHit]:
|
|
101
|
+
params: dict[str, Any] = {"tenant_id": tenant_id, "vector": _literal(vector), "limit": limit}
|
|
102
|
+
where = ["tenant_id = :tenant_id"]
|
|
103
|
+
for index, (key, value) in enumerate((filters or {}).items()):
|
|
104
|
+
where.append(f"metadata ->> :fk{index} = :fv{index}")
|
|
105
|
+
params[f"fk{index}"], params[f"fv{index}"] = key, str(value) # ->> yields text
|
|
106
|
+
rows = await self._run(
|
|
107
|
+
text(f"""
|
|
108
|
+
SELECT id, tenant_id, document_id, chunk_index, title, source, text, content_hash, metadata,
|
|
109
|
+
1 - (embedding <=> CAST(:vector AS vector)) AS score
|
|
110
|
+
FROM {TABLE} WHERE {" AND ".join(where)}
|
|
111
|
+
ORDER BY embedding <=> CAST(:vector AS vector), document_id, chunk_index LIMIT :limit
|
|
112
|
+
"""),
|
|
113
|
+
params,
|
|
114
|
+
)
|
|
115
|
+
hits = []
|
|
116
|
+
for row in rows:
|
|
117
|
+
fields = {k: v for k, v in row._mapping.items() if k != "score"}
|
|
118
|
+
fields["metadata"] = metadata_of(_json(row.metadata))
|
|
119
|
+
hits.append(SearchHit(chunk=Chunk(**fields), score=max(0.0, min(1.0, float(row.score)))))
|
|
120
|
+
return hits
|
|
121
|
+
|
|
122
|
+
async def _run(self, statement, params) -> list[Any]:
|
|
123
|
+
def run() -> list[Any]:
|
|
124
|
+
with self._session_factory() as session:
|
|
125
|
+
result = session.execute(statement, params)
|
|
126
|
+
rows = list(result) if result.returns_rows else []
|
|
127
|
+
session.commit()
|
|
128
|
+
return rows
|
|
129
|
+
|
|
130
|
+
return await asyncio.to_thread(run)
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _literal(vector: list[float]) -> str:
|
|
134
|
+
return "[" + ",".join(repr(float(v)) for v in vector) + "]"
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _json(value: Any) -> dict[str, Any]:
|
|
138
|
+
return json.loads(value) if isinstance(value, str) else dict(value or {})
|