@polymorphism-tech/morph-spec 4.16.0 → 4.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/bin/morph-spec.js +3 -0
- package/bin/task-manager.js +999 -962
- package/claude-plugin.json +1 -1
- package/docs/CHEATSHEET.md +1 -1
- package/docs/QUICKSTART.md +1 -1
- package/framework/agents/ai-agents/vector-search-expert.md +247 -233
- package/framework/agents/backend/ms-agent-expert.md +1 -1
- package/framework/agents/backend/neon-expert.md +397 -0
- package/framework/agents/frontend/gsap-animator.md +131 -0
- package/framework/agents.json +1175 -1085
- package/framework/commands/morph-preflight.md +200 -192
- package/framework/hooks/claude-code/session-start/inject-morph-context.js +1 -1
- package/framework/rules/gsap-standards.md +13 -0
- package/framework/rules/neon-standards.md +25 -0
- package/framework/skills/level-0-meta/mcp-registry.json +1 -1
- package/framework/skills/level-0-meta/morph-agent-teams/SKILL.md +472 -472
- package/framework/skills/level-0-meta/morph-init/SKILL.md +11 -11
- package/framework/skills/level-0-meta/morph-post-implementation/SKILL.md +5 -5
- package/framework/skills/level-0-meta/morph-tool-usage-guide/SKILL.md +5 -5
- package/framework/skills/level-0-meta/morph-tool-usage-guide/references/tools-per-phase.md +7 -7
- package/framework/skills/level-1-workflows/morph-phase-codebase-analysis/SKILL.md +24 -24
- package/framework/skills/level-1-workflows/morph-phase-design/SKILL.md +12 -12
- package/framework/skills/level-1-workflows/morph-phase-implement/SKILL.md +9 -9
- package/framework/skills/level-1-workflows/morph-phase-plan/SKILL.md +1 -1
- package/framework/skills/level-1-workflows/morph-phase-tasks/SKILL.md +1 -1
- package/framework/skills/level-1-workflows/morph-phase-tasks/references/task-planning-patterns.md +143 -143
- package/framework/skills/level-1-workflows/morph-phase-uiux/SKILL.md +1 -1
- package/framework/skills/level-1-workflows/morph-scope-escalation/SKILL.md +1 -1
- package/framework/standards/STANDARDS.json +184 -107
- package/framework/standards/ai-agents/production.md +1 -0
- package/framework/standards/ai-agents/setup.md +3 -2
- package/framework/standards/ai-agents/workflows.md +1 -0
- package/framework/standards/architecture/vertical-slice/vertical-slice.md +4 -4
- package/framework/standards/backend/database/postgresql/database.md +58 -78
- package/framework/standards/backend/dotnet/scalar-ui.md +172 -172
- package/framework/standards/backend/dotnet/vsa-handler-patterns.md +274 -274
- package/framework/standards/backend/integrations/asaas/asaas-api.md +1 -1
- package/framework/standards/backend/integrations/clerk/clerk-auth.md +23 -21
- package/framework/standards/frontend/design-system/animations.md +9 -0
- package/framework/standards/frontend/gsap/gsap-core.md +463 -0
- package/framework/standards/frontend/gsap/gsap-plugins.md +294 -0
- package/framework/standards/frontend/gsap/gsap-scrolltrigger.md +318 -0
- package/framework/standards/frontend/nextjs/motion-patterns.md +1 -0
- package/framework/standards/frontend/nextjs/nextjs-patterns.md +47 -37
- package/framework/standards/infrastructure/docker/easypanel-deploy.md +4 -8
- package/framework/standards/infrastructure/neon/neon-pgrag.md +239 -0
- package/framework/standards/infrastructure/neon/neon-pgvector.md +236 -0
- package/framework/standards/infrastructure/neon/neon-semantic-kernel.md +310 -0
- package/framework/standards/infrastructure/neon/neon-setup.md +186 -0
- package/framework/standards/integration/mcp/mcp-tools.md +32 -34
- package/framework/templates/README.md +18 -18
- package/framework/templates/REGISTRY.json +49 -96
- package/framework/templates/code/dotnet/abstractions/api-endpoint.cs +7 -7
- package/framework/templates/code/dotnet/backend/repository.cs +148 -148
- package/framework/templates/code/dotnet/contracts/contracts-vsa.cs +2 -2
- package/framework/templates/code/dotnet/database/vector-entity.cs +93 -0
- package/framework/templates/code/dotnet/feature/endpoint.cs.hbs +1 -1
- package/framework/templates/code/sql/{supabase-migration.sql → postgresql-migration.sql} +8 -17
- package/framework/templates/code/sql/{supabase-migration.template.sql → postgresql-migration.template.sql} +10 -13
- package/framework/templates/code/sql/{rls-policy.sql → postgresql-rls-policy.sql} +8 -8
- package/framework/templates/code/typescript/contracts.ts +167 -168
- package/framework/templates/context/CONTEXT.md +3 -3
- package/framework/templates/docs/proposal.md +6 -6
- package/framework/templates/docs/ui-components.md +1 -1
- package/framework/templates/feature/decisions.md +121 -121
- package/framework/templates/feature/recap.md +145 -145
- package/framework/templates/feature/tasks.md +199 -199
- package/framework/templates/frontend/nextjs/gsap-scroll-section.tsx.hbs +44 -0
- package/framework/templates/infrastructure/docker/docker-compose.template.yml +50 -67
- package/framework/templates/infrastructure/docker/dockerfile-api.dockerfile +38 -38
- package/framework/templates/infrastructure/docker/dockerfile-web.dockerfile +48 -48
- package/framework/templates/infrastructure/docker/easypanel.template.json +46 -54
- package/framework/templates/infrastructure/github/README.md +56 -56
- package/framework/templates/integrations/semantic-kernel-neon.cs +125 -0
- package/framework/workflows/README.md +1 -1
- package/framework/workflows/docs/full-morph.md +69 -69
- package/package.json +1 -1
- package/src/commands/agents/dispatch-agents.js +19 -15
- package/src/commands/state/advance-phase.js +5 -0
- package/src/commands/tasks/task.js +3 -0
- package/src/core/state/state-manager.js +45 -1
- package/src/core/templates/template-registry.js +1 -66
- package/src/lib/detectors/index.js +2 -2
- package/src/lib/generators/architecture-scanner.js +640 -640
- package/src/lib/installers/mcp-installer.js +1 -1
- package/src/lib/tasks/task-classifier.js +15 -1
- package/src/lib/tasks/test-runner.js +5 -1
- package/src/lib/validators/content/content-validator.js +9 -10
- package/src/lib/validators/validation-runner.js +7 -5
- package/framework/agents/backend/supabase-expert.md +0 -256
- package/framework/rules/supabase-standards.md +0 -29
- package/framework/standards/infrastructure/supabase/mcp-setup.md +0 -142
- package/framework/standards/infrastructure/supabase/supabase-auth.md +0 -176
- package/framework/standards/infrastructure/supabase/supabase-custom-jwt.md +0 -206
- package/framework/standards/infrastructure/supabase/supabase-pgvector.md +0 -169
- package/framework/standards/infrastructure/supabase/supabase-rls.md +0 -184
- package/framework/standards/infrastructure/supabase/supabase-storage.md +0 -153
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
# Next.js 15 Patterns Standard
|
|
2
2
|
|
|
3
|
-
> **Scope:** nextjs-
|
|
3
|
+
> **Scope:** nextjs-neon
|
|
4
4
|
> **Layer:** 2 (on keyword)
|
|
5
5
|
> **Keywords:** nextjs, react, app router, server component, client component
|
|
6
6
|
> **Load When:** nextjs keywords detected
|
|
7
7
|
|
|
8
|
-
Stack: Next.js 15 +
|
|
8
|
+
Stack: Next.js 15 + Neon PostgreSQL + .NET Backend
|
|
9
9
|
|
|
10
10
|
## Core Rules
|
|
11
11
|
|
|
12
12
|
- ALWAYS use App Router (not Pages Router)
|
|
13
13
|
- Default to Server Components -- add `'use client'` only when needed
|
|
14
14
|
- ALWAYS colocate loading.tsx and error.tsx with page.tsx
|
|
15
|
-
- NEVER call
|
|
15
|
+
- NEVER call the database directly from client -- use Route Handlers as BFF
|
|
16
16
|
- ALWAYS validate inputs with Zod on both client and server
|
|
17
17
|
- Use TypeScript strict mode (`"strict": true`)
|
|
18
18
|
|
|
@@ -64,39 +64,49 @@ export default function Error({ error, reset }: { error: Error; reset: () => voi
|
|
|
64
64
|
## Route Handlers (BFF Pattern)
|
|
65
65
|
|
|
66
66
|
```ts
|
|
67
|
-
// app/api/documents/route.ts
|
|
68
|
-
import {
|
|
67
|
+
// app/api/documents/route.ts — BFF pattern (delegates to .NET API)
|
|
68
|
+
import { auth } from "@clerk/nextjs/server";
|
|
69
69
|
import { NextResponse } from "next/server";
|
|
70
70
|
import { z } from "zod";
|
|
71
71
|
|
|
72
|
+
const API_URL = process.env.API_URL!; // .NET backend
|
|
73
|
+
|
|
72
74
|
const CreateSchema = z.object({
|
|
73
75
|
title: z.string().min(1).max(200),
|
|
74
76
|
content: z.string().min(1),
|
|
75
77
|
});
|
|
76
78
|
|
|
77
79
|
export async function GET() {
|
|
78
|
-
const
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
const
|
|
82
|
-
|
|
83
|
-
|
|
80
|
+
const { userId, getToken } = await auth();
|
|
81
|
+
if (!userId) return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
|
|
82
|
+
|
|
83
|
+
const token = await getToken();
|
|
84
|
+
const res = await fetch(`${API_URL}/api/documents`, {
|
|
85
|
+
headers: { Authorization: `Bearer ${token}` },
|
|
86
|
+
});
|
|
87
|
+
return NextResponse.json(await res.json());
|
|
84
88
|
}
|
|
85
89
|
|
|
86
90
|
export async function POST(request: Request) {
|
|
87
|
-
const
|
|
88
|
-
|
|
89
|
-
|
|
91
|
+
const { userId, getToken } = await auth();
|
|
92
|
+
if (!userId) return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
|
|
93
|
+
|
|
90
94
|
const parsed = CreateSchema.safeParse(await request.json());
|
|
91
95
|
if (!parsed.success) return NextResponse.json({ error: parsed.error.flatten() }, { status: 400 });
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
+
|
|
97
|
+
const token = await getToken();
|
|
98
|
+
const res = await fetch(`${API_URL}/api/documents`, {
|
|
99
|
+
method: "POST",
|
|
100
|
+
headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" },
|
|
101
|
+
body: JSON.stringify(parsed.data),
|
|
102
|
+
});
|
|
103
|
+
return NextResponse.json(await res.json(), { status: res.status });
|
|
96
104
|
}
|
|
97
105
|
```
|
|
98
106
|
|
|
99
|
-
|
|
107
|
+
> **Rule:** Route handlers act as BFF (Backend-For-Frontend) — they delegate to the .NET API. Database access lives in the .NET backend via EF Core, never in Next.js route handlers.
|
|
108
|
+
|
|
109
|
+
## React Query + Neon
|
|
100
110
|
|
|
101
111
|
```tsx
|
|
102
112
|
// providers/query-provider.tsx
|
|
@@ -190,26 +200,26 @@ Components are copied to `components/ui/` -- NOT an npm dependency, your code to
|
|
|
190
200
|
| Wrong | Right | Why |
|
|
191
201
|
|-------|-------|-----|
|
|
192
202
|
| `'use client'` on every component | Default to Server Components | Unnecessary JS bundle size |
|
|
193
|
-
| Direct
|
|
203
|
+
| Direct DB from client | Route Handler `/api/*` as BFF | Exposes queries, harder to secure |
|
|
194
204
|
| `any` for API responses | Zod schema + infer | No runtime safety |
|
|
195
205
|
| Missing loading.tsx | Colocate with page.tsx | Blank page during load |
|
|
196
206
|
| `useEffect` for data fetching | React Query `useQuery` | No caching, race conditions |
|
|
197
207
|
| Form validation on submit only | Zod resolver + react-hook-form | Delayed error feedback |
|
|
198
208
|
| shadcn as npm package | `npx shadcn@latest add` | Copy-paste system, not a dependency |
|
|
199
|
-
|
|
200
|
-
## TypeScript Strict Mode (Required)
|
|
201
|
-
|
|
202
|
-
Always enable `"strict": true` in `tsconfig.json`. This is non-negotiable for
|
|
203
|
-
agent-assisted development because agents rely on compiler errors to self-correct.
|
|
204
|
-
Without strict mode, type/null errors only surface at runtime where agents cannot observe them.
|
|
205
|
-
|
|
206
|
-
```json
|
|
207
|
-
{
|
|
208
|
-
"compilerOptions": {
|
|
209
|
-
"strict": true,
|
|
210
|
-
"noUncheckedIndexedAccess": true
|
|
211
|
-
}
|
|
212
|
-
}
|
|
213
|
-
```
|
|
214
|
-
|
|
215
|
-
Strict mode enables: `strictNullChecks`, `noImplicitAny`, `strictFunctionTypes`, `strictPropertyInitialization`.
|
|
209
|
+
|
|
210
|
+
## TypeScript Strict Mode (Required)
|
|
211
|
+
|
|
212
|
+
Always enable `"strict": true` in `tsconfig.json`. This is non-negotiable for
|
|
213
|
+
agent-assisted development because agents rely on compiler errors to self-correct.
|
|
214
|
+
Without strict mode, type/null errors only surface at runtime where agents cannot observe them.
|
|
215
|
+
|
|
216
|
+
```json
|
|
217
|
+
{
|
|
218
|
+
"compilerOptions": {
|
|
219
|
+
"strict": true,
|
|
220
|
+
"noUncheckedIndexedAccess": true
|
|
221
|
+
}
|
|
222
|
+
}
|
|
223
|
+
```
|
|
224
|
+
|
|
225
|
+
Strict mode enables: `strictNullChecks`, `noImplicitAny`, `strictFunctionTypes`, `strictPropertyInitialization`.
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
# EasyPanel Deployment Standard
|
|
2
2
|
|
|
3
|
-
> **Scope:** nextjs-
|
|
3
|
+
> **Scope:** nextjs-neon
|
|
4
4
|
> **Layer:** 2 (on keyword)
|
|
5
5
|
> **Keywords:** easypanel, docker, deploy, deployment, container
|
|
6
6
|
> **Load When:** easypanel or docker deployment keywords detected
|
|
7
7
|
|
|
8
|
-
Stack: Next.js 15 +
|
|
8
|
+
Stack: Next.js 15 + Neon PostgreSQL + .NET Backend
|
|
9
9
|
|
|
10
10
|
## Core Rules
|
|
11
11
|
|
|
@@ -115,17 +115,13 @@ SSL via Let's Encrypt is automatic. Force HTTPS enabled by default.
|
|
|
115
115
|
```env
|
|
116
116
|
ASPNETCORE_ENVIRONMENT=Production
|
|
117
117
|
ASPNETCORE_URLS=http://+:8080
|
|
118
|
-
ConnectionStrings__DefaultConnection=Host
|
|
119
|
-
Supabase__Url=https://xxx.supabase.co
|
|
120
|
-
Supabase__ServiceRoleKey=eyJ...
|
|
121
|
-
Supabase__JwtSecret=your-jwt-secret
|
|
118
|
+
ConnectionStrings__DefaultConnection=Host=ep-xxx.us-east-2.aws.neon.tech;Database=neondb;Username=...;Password=...;SslMode=Require
|
|
122
119
|
```
|
|
123
120
|
|
|
124
121
|
### Next.js Frontend
|
|
125
122
|
|
|
126
123
|
```env
|
|
127
|
-
|
|
128
|
-
NEXT_PUBLIC_SUPABASE_ANON_KEY=eyJ...
|
|
124
|
+
DATABASE_URL=postgresql://user:pass@ep-xxx.us-east-2.aws.neon.tech/neondb?sslmode=require
|
|
129
125
|
NEXT_PUBLIC_API_URL=https://api.example.com
|
|
130
126
|
```
|
|
131
127
|
|
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
# Neon pgrag Standard
|
|
2
|
+
|
|
3
|
+
> **Scope:** neon
|
|
4
|
+
> **Layer:** 2 (on keyword)
|
|
5
|
+
> **Keywords:** pgrag, rag, chunking, embedding, reranking, retrieval
|
|
6
|
+
> **Load When:** pgrag or in-database RAG keywords detected
|
|
7
|
+
|
|
8
|
+
Stack: Neon Serverless Postgres
|
|
9
|
+
|
|
10
|
+
## Core Rules
|
|
11
|
+
|
|
12
|
+
- pgrag is EXPERIMENTAL -- API may change between Neon releases
|
|
13
|
+
- ALWAYS enable unstable extensions before installing: `SET neon.allow_unstable_extensions='true'`
|
|
14
|
+
- Use pgrag for simple, self-contained RAG pipelines (PDF ingestion, chunking, search)
|
|
15
|
+
- Use application-level RAG (Semantic Kernel + pgvector) for complex workflows requiring custom logic, multi-model orchestration, or .NET integration
|
|
16
|
+
- NEVER use pgrag local embeddings (384 dims) when your application uses a different embedding model -- dimension mismatch breaks search
|
|
17
|
+
|
|
18
|
+
## Install pgrag
|
|
19
|
+
|
|
20
|
+
```sql
|
|
21
|
+
-- Enable unstable extensions (required)
|
|
22
|
+
SET neon.allow_unstable_extensions='true';
|
|
23
|
+
|
|
24
|
+
-- Core extension
|
|
25
|
+
CREATE EXTENSION IF NOT EXISTS rag CASCADE;
|
|
26
|
+
|
|
27
|
+
-- Local embedding model (bge-small-en-v1.5, 384 dimensions, 33M params)
|
|
28
|
+
CREATE EXTENSION IF NOT EXISTS rag_bge_small_en_v15 CASCADE;
|
|
29
|
+
|
|
30
|
+
-- Local reranker (jina-reranker-v1-tiny-en, 33M params)
|
|
31
|
+
CREATE EXTENSION IF NOT EXISTS rag_jina_reranker_v1_tiny_en CASCADE;
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
## Text Extraction
|
|
35
|
+
|
|
36
|
+
Extract text from binary documents directly in SQL:
|
|
37
|
+
|
|
38
|
+
```sql
|
|
39
|
+
-- Extract text from PDF
|
|
40
|
+
SELECT rag.text_from_pdf(pg_read_binary_file('/path/to/document.pdf'));
|
|
41
|
+
|
|
42
|
+
-- Extract text from DOCX
|
|
43
|
+
SELECT rag.text_from_docx(pg_read_binary_file('/path/to/document.docx'));
|
|
44
|
+
|
|
45
|
+
-- Convert HTML to Markdown
|
|
46
|
+
SELECT rag.markdown_from_html('<h1>Title</h1><p>Content here</p>');
|
|
47
|
+
```
|
|
48
|
+
|
|
49
|
+
### From a Documents Table
|
|
50
|
+
|
|
51
|
+
```sql
|
|
52
|
+
CREATE TABLE raw_documents (
|
|
53
|
+
id SERIAL PRIMARY KEY,
|
|
54
|
+
filename TEXT NOT NULL,
|
|
55
|
+
content BYTEA NOT NULL,
|
|
56
|
+
extracted_text TEXT
|
|
57
|
+
);
|
|
58
|
+
|
|
59
|
+
-- Extract text on insert
|
|
60
|
+
UPDATE raw_documents
|
|
61
|
+
SET extracted_text = rag.text_from_pdf(content)
|
|
62
|
+
WHERE filename LIKE '%.pdf' AND extracted_text IS NULL;
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
## Chunking Strategies
|
|
66
|
+
|
|
67
|
+
### Character-Based Chunking
|
|
68
|
+
|
|
69
|
+
Splits text by character count with overlap for context continuity:
|
|
70
|
+
|
|
71
|
+
```sql
|
|
72
|
+
-- Split into chunks of 1000 chars with 200 char overlap
|
|
73
|
+
SELECT unnest(
|
|
74
|
+
rag.chunks_by_character_count(
|
|
75
|
+
'Your long document text here...',
|
|
76
|
+
1000, -- max_chars per chunk
|
|
77
|
+
200 -- overlap chars
|
|
78
|
+
)
|
|
79
|
+
);
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
### Token-Based Chunking
|
|
83
|
+
|
|
84
|
+
Uses the bge-small-en-v1.5 tokenizer for model-aware splitting:
|
|
85
|
+
|
|
86
|
+
```sql
|
|
87
|
+
-- Split into chunks of 512 tokens with 50 token overlap
|
|
88
|
+
SELECT unnest(
|
|
89
|
+
rag_bge_small_en_v15.chunks_by_token_count(
|
|
90
|
+
'Your long document text here...',
|
|
91
|
+
512, -- max_tokens per chunk
|
|
92
|
+
50 -- overlap tokens
|
|
93
|
+
)
|
|
94
|
+
);
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
## Embedding Generation
|
|
98
|
+
|
|
99
|
+
### Local Embeddings (bge-small-en-v1.5)
|
|
100
|
+
|
|
101
|
+
Zero-cost, runs on the database server. Produces 384-dimension vectors:
|
|
102
|
+
|
|
103
|
+
```sql
|
|
104
|
+
-- Embedding for a document passage (includes "passage:" prefix internally)
|
|
105
|
+
SELECT rag_bge_small_en_v15.embedding_for_passage('Neon is a serverless Postgres platform');
|
|
106
|
+
|
|
107
|
+
-- Embedding for a search query (includes "query:" prefix internally)
|
|
108
|
+
SELECT rag_bge_small_en_v15.embedding_for_query('What is Neon?');
|
|
109
|
+
```
|
|
110
|
+
|
|
111
|
+
### OpenAI Embeddings (text-embedding-3-small)
|
|
112
|
+
|
|
113
|
+
Remote API call, produces 1536-dimension vectors:
|
|
114
|
+
|
|
115
|
+
```sql
|
|
116
|
+
-- Set API key (per session)
|
|
117
|
+
SET neon.ai_api_key = 'sk-...';
|
|
118
|
+
|
|
119
|
+
-- Generate embedding via OpenAI
|
|
120
|
+
SELECT rag.openai_text_embedding_3_small('Neon is a serverless Postgres platform');
|
|
121
|
+
```
|
|
122
|
+
|
|
123
|
+
## Full RAG Pipeline Example
|
|
124
|
+
|
|
125
|
+
```sql
|
|
126
|
+
-- 1. Create chunks table
|
|
127
|
+
CREATE TABLE chunks (
|
|
128
|
+
id SERIAL PRIMARY KEY,
|
|
129
|
+
document_id INT REFERENCES raw_documents(id),
|
|
130
|
+
chunk_text TEXT NOT NULL,
|
|
131
|
+
embedding vector(384)
|
|
132
|
+
);
|
|
133
|
+
|
|
134
|
+
-- 2. Chunk and embed a document
|
|
135
|
+
WITH doc AS (
|
|
136
|
+
SELECT id, rag.text_from_pdf(content) AS full_text
|
|
137
|
+
FROM raw_documents WHERE id = 1
|
|
138
|
+
),
|
|
139
|
+
chunked AS (
|
|
140
|
+
SELECT doc.id AS document_id, unnest(
|
|
141
|
+
rag.chunks_by_character_count(doc.full_text, 1000, 200)
|
|
142
|
+
) AS chunk_text
|
|
143
|
+
FROM doc
|
|
144
|
+
)
|
|
145
|
+
INSERT INTO chunks (document_id, chunk_text, embedding)
|
|
146
|
+
SELECT document_id, chunk_text,
|
|
147
|
+
rag_bge_small_en_v15.embedding_for_passage(chunk_text)
|
|
148
|
+
FROM chunked;
|
|
149
|
+
|
|
150
|
+
-- 3. Create HNSW index
|
|
151
|
+
CREATE INDEX idx_chunks_embedding ON chunks
|
|
152
|
+
USING hnsw (embedding vector_cosine_ops) WITH (m = 16, ef_construction = 64);
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
## Similarity Search
|
|
156
|
+
|
|
157
|
+
```sql
|
|
158
|
+
-- Find top 10 similar chunks
|
|
159
|
+
SELECT id, chunk_text,
|
|
160
|
+
1 - (embedding <=> rag_bge_small_en_v15.embedding_for_query('What is Neon?')) AS similarity
|
|
161
|
+
FROM chunks
|
|
162
|
+
ORDER BY embedding <=> rag_bge_small_en_v15.embedding_for_query('What is Neon?')
|
|
163
|
+
LIMIT 10;
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
## Reranking
|
|
167
|
+
|
|
168
|
+
Reranking refines initial vector search results using a cross-encoder model for higher accuracy:
|
|
169
|
+
|
|
170
|
+
```sql
|
|
171
|
+
-- Rerank top 10 vector results down to top 5
|
|
172
|
+
WITH vector_matches AS (
|
|
173
|
+
SELECT id, chunk_text,
|
|
174
|
+
1 - (embedding <=> rag_bge_small_en_v15.embedding_for_query('What is Neon?')) AS similarity
|
|
175
|
+
FROM chunks
|
|
176
|
+
ORDER BY embedding <=> rag_bge_small_en_v15.embedding_for_query('What is Neon?')
|
|
177
|
+
LIMIT 10
|
|
178
|
+
)
|
|
179
|
+
SELECT id, chunk_text, similarity,
|
|
180
|
+
rag_jina_reranker_v1_tiny_en.rerank_distance('What is Neon?', chunk_text) AS rerank_score
|
|
181
|
+
FROM vector_matches
|
|
182
|
+
ORDER BY rerank_score ASC
|
|
183
|
+
LIMIT 5;
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
## LLM Integration in SQL
|
|
187
|
+
|
|
188
|
+
Send context + question to an LLM directly from SQL:
|
|
189
|
+
|
|
190
|
+
```sql
|
|
191
|
+
-- Set API key
|
|
192
|
+
SET neon.ai_api_key = 'sk-...';
|
|
193
|
+
|
|
194
|
+
-- Ask a question with retrieved context
|
|
195
|
+
WITH context AS (
|
|
196
|
+
SELECT string_agg(chunk_text, E'\n---\n') AS combined_context
|
|
197
|
+
FROM (
|
|
198
|
+
SELECT chunk_text FROM chunks
|
|
199
|
+
ORDER BY embedding <=> rag_bge_small_en_v15.embedding_for_query('What is Neon?')
|
|
200
|
+
LIMIT 5
|
|
201
|
+
) top_chunks
|
|
202
|
+
)
|
|
203
|
+
SELECT rag.openai_chat_completion(
|
|
204
|
+
json_build_object(
|
|
205
|
+
'model', 'gpt-4o',
|
|
206
|
+
'messages', json_build_array(
|
|
207
|
+
json_build_object('role', 'system', 'content',
|
|
208
|
+
'Answer based on the provided context. If unsure, say so.'),
|
|
209
|
+
json_build_object('role', 'user', 'content',
|
|
210
|
+
format('Context:\n%s\n\nQuestion: What is Neon?', context.combined_context))
|
|
211
|
+
)
|
|
212
|
+
)
|
|
213
|
+
) FROM context;
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
## When to Use pgrag vs Application-Level RAG
|
|
217
|
+
|
|
218
|
+
| Factor | pgrag (In-Database) | SK + pgvector (Application) |
|
|
219
|
+
|--------|---------------------|----------------------------|
|
|
220
|
+
| Setup complexity | Minimal -- SQL only | Requires .NET service layer |
|
|
221
|
+
| Embedding model | Local (384d) or OpenAI | Any model via SK connectors |
|
|
222
|
+
| Reranking | Built-in (Jina) | Custom implementation |
|
|
223
|
+
| Custom logic | Limited to SQL | Full C# flexibility |
|
|
224
|
+
| Multi-model orchestration | Not supported | Native SK capability |
|
|
225
|
+
| Production readiness | Experimental | Stable (SK GA) |
|
|
226
|
+
| Cost | Zero for local models | API costs for remote models |
|
|
227
|
+
| Best for | Prototyping, simple RAG, SQL-only environments | Production .NET applications, complex pipelines |
|
|
228
|
+
|
|
229
|
+
**Recommendation:** Start with pgrag for prototyping and validation. Migrate to Semantic Kernel + pgvector for production .NET applications that need custom retrieval logic, multi-model orchestration, or integration with MAF agents.
|
|
230
|
+
|
|
231
|
+
## Common Mistakes
|
|
232
|
+
|
|
233
|
+
| Wrong | Right | Why |
|
|
234
|
+
|-------|-------|-----|
|
|
235
|
+
| Missing `neon.allow_unstable_extensions` | `SET neon.allow_unstable_extensions='true'` | pgrag won't install without it |
|
|
236
|
+
| Mixing local (384d) and OpenAI (1536d) embeddings | Use one model consistently per column | Dimension mismatch breaks similarity search |
|
|
237
|
+
| No HNSW index on chunks table | Create index after initial data load | Full table scan on every search |
|
|
238
|
+
| Using pgrag in production without testing | Validate with representative data first | Extension is experimental, may change |
|
|
239
|
+
| Storing API key in SQL scripts | Use session variables or environment config | Keys in scripts leak to version control |
|
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
# Neon pgvector Standard
|
|
2
|
+
|
|
3
|
+
> **Scope:** neon, dotnet-vsa
|
|
4
|
+
> **Layer:** 1 (on scope)
|
|
5
|
+
> **Keywords:** pgvector, vector, embeddings, similarity search, HNSW, cosine
|
|
6
|
+
> **Load When:** pgvector or embedding keywords detected
|
|
7
|
+
|
|
8
|
+
Stack: .NET VSA + Neon Serverless Postgres
|
|
9
|
+
|
|
10
|
+
## Core Rules
|
|
11
|
+
|
|
12
|
+
- ALWAYS use HNSW indexes for production (faster queries, no training data required)
|
|
13
|
+
- ALWAYS match vector dimensions to your embedding model (e.g., 1536 for text-embedding-3-small)
|
|
14
|
+
- NEVER store embeddings without an index -- full table scan at query time
|
|
15
|
+
- ALWAYS call `UseVector()` on `NpgsqlDataSourceBuilder` before using vector types
|
|
16
|
+
- ALWAYS register the `vector` extension in `OnModelCreating`
|
|
17
|
+
- Use `ReadOnlyMemory<float>` for vector properties in EF Core entities
|
|
18
|
+
|
|
19
|
+
## Enable pgvector
|
|
20
|
+
|
|
21
|
+
```sql
|
|
22
|
+
CREATE EXTENSION IF NOT EXISTS vector;
|
|
23
|
+
```
|
|
24
|
+
|
|
25
|
+
## EF Core Entity Mapping
|
|
26
|
+
|
|
27
|
+
### NuGet Package
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
dotnet add package Pgvector.EntityFrameworkCore
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
### Entity Definition
|
|
34
|
+
|
|
35
|
+
```csharp
|
|
36
|
+
using System.ComponentModel.DataAnnotations.Schema;
|
|
37
|
+
using Pgvector;
|
|
38
|
+
|
|
39
|
+
public class Document
|
|
40
|
+
{
|
|
41
|
+
public Guid Id { get; set; }
|
|
42
|
+
public string Title { get; set; } = string.Empty;
|
|
43
|
+
public string Content { get; set; } = string.Empty;
|
|
44
|
+
public string? Metadata { get; set; }
|
|
45
|
+
|
|
46
|
+
[Column(TypeName = "vector(1536)")]
|
|
47
|
+
public Vector? Embedding { get; set; }
|
|
48
|
+
|
|
49
|
+
public DateTimeOffset CreatedAt { get; set; } = DateTimeOffset.UtcNow;
|
|
50
|
+
}
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
### IEntityTypeConfiguration
|
|
54
|
+
|
|
55
|
+
```csharp
|
|
56
|
+
using Microsoft.EntityFrameworkCore;
|
|
57
|
+
using Microsoft.EntityFrameworkCore.Metadata.Builders;
|
|
58
|
+
|
|
59
|
+
public class DocumentConfiguration : IEntityTypeConfiguration<Document>
|
|
60
|
+
{
|
|
61
|
+
public void Configure(EntityTypeBuilder<Document> builder)
|
|
62
|
+
{
|
|
63
|
+
builder.ToTable("documents");
|
|
64
|
+
builder.HasKey(d => d.Id);
|
|
65
|
+
builder.Property(d => d.Id).HasDefaultValueSql("gen_random_uuid()");
|
|
66
|
+
builder.Property(d => d.Title).IsRequired();
|
|
67
|
+
builder.Property(d => d.Content).IsRequired();
|
|
68
|
+
builder.Property(d => d.Metadata).HasColumnType("jsonb");
|
|
69
|
+
builder.Property(d => d.Embedding).HasColumnType("vector(1536)");
|
|
70
|
+
builder.Property(d => d.CreatedAt).HasDefaultValueSql("now()");
|
|
71
|
+
|
|
72
|
+
// HNSW index for cosine similarity
|
|
73
|
+
builder.HasIndex(d => d.Embedding)
|
|
74
|
+
.HasMethod("hnsw")
|
|
75
|
+
.HasOperators("vector_cosine_ops")
|
|
76
|
+
.HasStorageParameter("m", 16)
|
|
77
|
+
.HasStorageParameter("ef_construction", 64);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
### DbContext Setup
|
|
83
|
+
|
|
84
|
+
```csharp
|
|
85
|
+
public class AppDbContext : DbContext
|
|
86
|
+
{
|
|
87
|
+
public DbSet<Document> Documents => Set<Document>();
|
|
88
|
+
|
|
89
|
+
public AppDbContext(DbContextOptions<AppDbContext> options) : base(options) { }
|
|
90
|
+
|
|
91
|
+
protected override void OnModelCreating(ModelBuilder modelBuilder)
|
|
92
|
+
{
|
|
93
|
+
modelBuilder.HasPostgresExtension("vector");
|
|
94
|
+
modelBuilder.ApplyConfiguration(new DocumentConfiguration());
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// Registration with vector support
|
|
99
|
+
builder.Services.AddDbContext<AppDbContext>(options =>
|
|
100
|
+
options.UseNpgsql(connectionString, npgsql => npgsql.UseVector()));
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
## HNSW Index Creation (Raw SQL)
|
|
104
|
+
|
|
105
|
+
```sql
|
|
106
|
+
-- Cosine distance (most common for normalized embeddings)
|
|
107
|
+
CREATE INDEX idx_documents_embedding ON documents
|
|
108
|
+
USING hnsw (embedding vector_cosine_ops) WITH (m = 16, ef_construction = 64);
|
|
109
|
+
|
|
110
|
+
-- L2 (Euclidean) distance
|
|
111
|
+
CREATE INDEX idx_documents_embedding_l2 ON documents
|
|
112
|
+
USING hnsw (embedding vector_l2_ops) WITH (m = 16, ef_construction = 64);
|
|
113
|
+
|
|
114
|
+
-- Tune search accuracy per session
|
|
115
|
+
SET hnsw.ef_search = 100;
|
|
116
|
+
```
|
|
117
|
+
|
|
118
|
+
### HNSW Parameters
|
|
119
|
+
|
|
120
|
+
| Parameter | Default | Tuning |
|
|
121
|
+
|-----------|---------|--------|
|
|
122
|
+
| `m` | 16 | Higher = better recall, more memory |
|
|
123
|
+
| `ef_construction` | 64 | Higher = better index quality, slower build |
|
|
124
|
+
| `ef_search` | 40 | Higher = better recall at query time |
|
|
125
|
+
|
|
126
|
+
## Distance Operators
|
|
127
|
+
|
|
128
|
+
| Operator | Function | Index Ops | Use Case |
|
|
129
|
+
|----------|----------|-----------|----------|
|
|
130
|
+
| `<=>` | Cosine distance | `vector_cosine_ops` | Normalized embeddings (most common) |
|
|
131
|
+
| `<->` | L2 (Euclidean) | `vector_l2_ops` | Spatial/positional data |
|
|
132
|
+
| `<#>` | Inner product (neg) | `vector_ip_ops` | Pre-normalized, max similarity |
|
|
133
|
+
|
|
134
|
+
## Similarity Search
|
|
135
|
+
|
|
136
|
+
### SQL Function
|
|
137
|
+
|
|
138
|
+
```sql
|
|
139
|
+
CREATE OR REPLACE FUNCTION match_documents(
|
|
140
|
+
query_embedding vector(1536),
|
|
141
|
+
match_threshold float DEFAULT 0.78,
|
|
142
|
+
match_count int DEFAULT 10
|
|
143
|
+
) RETURNS TABLE (id uuid, title text, content text, similarity float)
|
|
144
|
+
LANGUAGE sql STABLE AS $$
|
|
145
|
+
SELECT d.id, d.title, d.content,
|
|
146
|
+
1 - (d.embedding <=> query_embedding) AS similarity
|
|
147
|
+
FROM documents d
|
|
148
|
+
WHERE 1 - (d.embedding <=> query_embedding) > match_threshold
|
|
149
|
+
ORDER BY d.embedding <=> query_embedding
|
|
150
|
+
LIMIT match_count;
|
|
151
|
+
$$;
|
|
152
|
+
```
|
|
153
|
+
|
|
154
|
+
### C# Repository
|
|
155
|
+
|
|
156
|
+
```csharp
|
|
157
|
+
using Pgvector;
|
|
158
|
+
|
|
159
|
+
public sealed class DocumentRepository(AppDbContext db)
|
|
160
|
+
{
|
|
161
|
+
public async Task StoreEmbeddingAsync(
|
|
162
|
+
Guid documentId, float[] embedding, CancellationToken ct = default)
|
|
163
|
+
{
|
|
164
|
+
await db.Database.ExecuteSqlInterpolatedAsync(
|
|
165
|
+
$"UPDATE documents SET embedding = {new Vector(embedding)} WHERE id = {documentId}", ct);
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
public async Task<List<DocumentMatch>> SearchSimilarAsync(
|
|
169
|
+
float[] queryEmbedding, int limit = 10, float threshold = 0.78f,
|
|
170
|
+
CancellationToken ct = default)
|
|
171
|
+
{
|
|
172
|
+
return await db.Database.SqlQuery<DocumentMatch>($"""
|
|
173
|
+
SELECT id, title, content,
|
|
174
|
+
1 - (embedding <=> {new Vector(queryEmbedding)}::vector) AS similarity
|
|
175
|
+
FROM documents
|
|
176
|
+
WHERE 1 - (embedding <=> {new Vector(queryEmbedding)}::vector) > {threshold}
|
|
177
|
+
ORDER BY embedding <=> {new Vector(queryEmbedding)}::vector
|
|
178
|
+
LIMIT {limit}
|
|
179
|
+
""").ToListAsync(ct);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
public record DocumentMatch(Guid Id, string Title, string Content, float Similarity);
|
|
184
|
+
```
|
|
185
|
+
|
|
186
|
+
## Hybrid Search (Vector + Full-Text)
|
|
187
|
+
|
|
188
|
+
```sql
|
|
189
|
+
CREATE OR REPLACE FUNCTION hybrid_search(
|
|
190
|
+
query_text text,
|
|
191
|
+
query_embedding vector(1536),
|
|
192
|
+
match_count int DEFAULT 10,
|
|
193
|
+
text_weight float DEFAULT 0.3,
|
|
194
|
+
vector_weight float DEFAULT 0.7
|
|
195
|
+
) RETURNS TABLE (id uuid, title text, content text, score float)
|
|
196
|
+
LANGUAGE sql STABLE AS $$
|
|
197
|
+
WITH vector_results AS (
|
|
198
|
+
SELECT id, title, content,
|
|
199
|
+
1 - (embedding <=> query_embedding) AS vector_score
|
|
200
|
+
FROM documents
|
|
201
|
+
ORDER BY embedding <=> query_embedding LIMIT match_count * 2
|
|
202
|
+
),
|
|
203
|
+
text_results AS (
|
|
204
|
+
SELECT id, title, content,
|
|
205
|
+
ts_rank(to_tsvector('english', content), plainto_tsquery('english', query_text)) AS text_score
|
|
206
|
+
FROM documents
|
|
207
|
+
WHERE to_tsvector('english', content) @@ plainto_tsquery('english', query_text)
|
|
208
|
+
LIMIT match_count * 2
|
|
209
|
+
)
|
|
210
|
+
SELECT COALESCE(v.id, t.id), COALESCE(v.title, t.title), COALESCE(v.content, t.content),
|
|
211
|
+
(COALESCE(v.vector_score, 0) * vector_weight + COALESCE(t.text_score, 0) * text_weight) AS score
|
|
212
|
+
FROM vector_results v FULL OUTER JOIN text_results t ON v.id = t.id
|
|
213
|
+
ORDER BY score DESC LIMIT match_count;
|
|
214
|
+
$$;
|
|
215
|
+
```
|
|
216
|
+
|
|
217
|
+
## Npgsql Vector Type Handler
|
|
218
|
+
|
|
219
|
+
The `Pgvector.EntityFrameworkCore` package automatically registers the vector type handler when you call `UseVector()`. For raw Npgsql (without EF Core):
|
|
220
|
+
|
|
221
|
+
```csharp
|
|
222
|
+
var dataSourceBuilder = new NpgsqlDataSourceBuilder(connectionString);
|
|
223
|
+
dataSourceBuilder.UseVector();
|
|
224
|
+
await using var dataSource = dataSourceBuilder.Build();
|
|
225
|
+
```
|
|
226
|
+
|
|
227
|
+
## Common Mistakes
|
|
228
|
+
|
|
229
|
+
| Wrong | Right | Why |
|
|
230
|
+
|-------|-------|-----|
|
|
231
|
+
| No index on embedding column | HNSW index | Full table scan, extremely slow |
|
|
232
|
+
| `ORDER BY similarity DESC` | `ORDER BY embedding <=> query ASC` | Operators return distance, not similarity |
|
|
233
|
+
| Missing `UseVector()` | Call on `NpgsqlDataSourceBuilder` | Npgsql cannot map vector types without it |
|
|
234
|
+
| Mixing embedding dimensions | Consistent dimensions per column | Dimension mismatch causes runtime errors |
|
|
235
|
+
| `float[]` for EF Core property | `Vector` from Pgvector package | EF Core mapping requires the Pgvector type |
|
|
236
|
+
| No `HasPostgresExtension("vector")` | Add in `OnModelCreating` | Extension must be declared for migrations |
|