@payloadcms-vectorize/cf 0.6.0-beta
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +210 -0
- package/dist/embed.js +27 -0
- package/dist/embed.js.map +1 -0
- package/dist/index.js +81 -0
- package/dist/index.js.map +1 -0
- package/dist/search.js +86 -0
- package/dist/search.js.map +1 -0
- package/dist/types.js +5 -0
- package/dist/types.js.map +1 -0
- package/package.json +23 -0
package/README.md
ADDED
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
# @payloadcms-vectorize/cf
|
|
2
|
+
|
|
3
|
+
Cloudflare Vectorize adapter for [payloadcms-vectorize](https://github.com/techiejd/payloadcms-vectorize). Enables vector search capabilities using Cloudflare Vectorize.
|
|
4
|
+
|
|
5
|
+
## Prerequisites
|
|
6
|
+
|
|
7
|
+
- Cloudflare account with Vectorize index configured
|
|
8
|
+
- Payload CMS 3.x with any supported database adapter
|
|
9
|
+
- Node.js 18+
|
|
10
|
+
|
|
11
|
+
## Installation
|
|
12
|
+
|
|
13
|
+
```bash
|
|
14
|
+
pnpm add @payloadcms-vectorize/cf payloadcms-vectorize
|
|
15
|
+
```
|
|
16
|
+
|
|
17
|
+
## Quick Start
|
|
18
|
+
|
|
19
|
+
### 1. Create Vectorize Index
|
|
20
|
+
|
|
21
|
+
Create a Vectorize index in your Cloudflare dashboard or via Wrangler:
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
wrangler vectorize create my-vectorize-index --dimensions=384 --metric=cosine
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
### 2. Configure the Plugin
|
|
28
|
+
|
|
29
|
+
```typescript
|
|
30
|
+
import { buildConfig } from 'payload'
|
|
31
|
+
import { postgresAdapter } from '@payloadcms/db-postgres'
|
|
32
|
+
import { createCloudflareVectorizeIntegration } from '@payloadcms-vectorize/cf'
|
|
33
|
+
import payloadcmsVectorize from 'payloadcms-vectorize'
|
|
34
|
+
|
|
35
|
+
// Create the integration
|
|
36
|
+
const integration = createCloudflareVectorizeIntegration({
|
|
37
|
+
config: {
|
|
38
|
+
default: {
|
|
39
|
+
dims: 384, // Vector dimensions (must match your embedding model and Vectorize index)
|
|
40
|
+
},
|
|
41
|
+
},
|
|
42
|
+
binding: env.VECTORIZE, // Cloudflare Vectorize binding
|
|
43
|
+
})
|
|
44
|
+
|
|
45
|
+
export default buildConfig({
|
|
46
|
+
// ... your existing config
|
|
47
|
+
db: postgresAdapter({
|
|
48
|
+
pool: {
|
|
49
|
+
connectionString: process.env.DATABASE_URL,
|
|
50
|
+
},
|
|
51
|
+
}),
|
|
52
|
+
plugins: [
|
|
53
|
+
payloadcmsVectorize({
|
|
54
|
+
dbAdapter: integration.adapter,
|
|
55
|
+
knowledgePools: {
|
|
56
|
+
default: {
|
|
57
|
+
collections: {
|
|
58
|
+
posts: {
|
|
59
|
+
toKnowledgePool: async (doc) => [{ chunk: doc.title || '' }],
|
|
60
|
+
},
|
|
61
|
+
},
|
|
62
|
+
embeddingConfig: {
|
|
63
|
+
version: 'v1.0.0',
|
|
64
|
+
queryFn: embedQuery,
|
|
65
|
+
realTimeIngestionFn: embedDocs,
|
|
66
|
+
},
|
|
67
|
+
},
|
|
68
|
+
},
|
|
69
|
+
}),
|
|
70
|
+
],
|
|
71
|
+
})
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
## Configuration
|
|
75
|
+
|
|
76
|
+
The `createCloudflareVectorizeIntegration` function accepts a configuration object with `config` and `binding` properties:
|
|
77
|
+
|
|
78
|
+
```typescript
|
|
79
|
+
const integration = createCloudflareVectorizeIntegration({
|
|
80
|
+
config: {
|
|
81
|
+
poolName: {
|
|
82
|
+
dims: number, // Required: Vector dimensions
|
|
83
|
+
},
|
|
84
|
+
// ... additional pools
|
|
85
|
+
},
|
|
86
|
+
binding: vectorizeBinding, // Required: Cloudflare Vectorize binding
|
|
87
|
+
})
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
### Configuration Options
|
|
91
|
+
|
|
92
|
+
| Option | Type | Required | Description |
|
|
93
|
+
| ------ | -------- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
94
|
+
| `dims` | `number` | Yes | Vector dimensions for the Vectorize index. Must match your embedding model's output dimensions and your Cloudflare Vectorize index configuration. |
|
|
95
|
+
|
|
96
|
+
### Cloudflare Bindings
|
|
97
|
+
|
|
98
|
+
| Property | Type | Required | Description |
|
|
99
|
+
| ----------- | ---------------- | -------- | ------------------------------------------------------------------------------------------------- |
|
|
100
|
+
| `vectorize` | `VectorizeIndex` | Yes | Cloudflare Vectorize binding for vector storage. Configured in `wrangler.toml` for Workers/Pages. |
|
|
101
|
+
|
|
102
|
+
## Integration Return Value
|
|
103
|
+
|
|
104
|
+
`createCloudflareVectorizeIntegration` returns an object with:
|
|
105
|
+
|
|
106
|
+
| Property | Type | Description |
|
|
107
|
+
| --------- | ----------- | ------------------------------------------------------------------------- |
|
|
108
|
+
| `adapter` | `DbAdapter` | The database adapter to pass to `payloadcmsVectorize({ dbAdapter: ... })` |
|
|
109
|
+
|
|
110
|
+
## Multiple Knowledge Pools
|
|
111
|
+
|
|
112
|
+
You can configure multiple knowledge pools with different dimensions:
|
|
113
|
+
|
|
114
|
+
```typescript
|
|
115
|
+
const integration = createCloudflareVectorizeIntegration({
|
|
116
|
+
config: {
|
|
117
|
+
documents: {
|
|
118
|
+
dims: 1536,
|
|
119
|
+
},
|
|
120
|
+
images: {
|
|
121
|
+
dims: 512,
|
|
122
|
+
},
|
|
123
|
+
},
|
|
124
|
+
binding: env.VECTORIZE,
|
|
125
|
+
})
|
|
126
|
+
|
|
127
|
+
export default buildConfig({
|
|
128
|
+
// ...
|
|
129
|
+
plugins: [
|
|
130
|
+
payloadcmsVectorize({
|
|
131
|
+
dbAdapter: integration.adapter,
|
|
132
|
+
knowledgePools: {
|
|
133
|
+
documents: {
|
|
134
|
+
collections: {
|
|
135
|
+
/* ... */
|
|
136
|
+
},
|
|
137
|
+
embeddingConfig: {
|
|
138
|
+
/* ... */
|
|
139
|
+
},
|
|
140
|
+
},
|
|
141
|
+
images: {
|
|
142
|
+
collections: {
|
|
143
|
+
/* ... */
|
|
144
|
+
},
|
|
145
|
+
embeddingConfig: {
|
|
146
|
+
/* ... */
|
|
147
|
+
},
|
|
148
|
+
},
|
|
149
|
+
},
|
|
150
|
+
}),
|
|
151
|
+
],
|
|
152
|
+
})
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
**Note:** Each knowledge pool requires a separate Vectorize index with matching dimensions.
|
|
156
|
+
|
|
157
|
+
## Using with Cloudflare AI
|
|
158
|
+
|
|
159
|
+
```typescript
|
|
160
|
+
export const embedDocs = async (texts: string[]): Promise<number[][]> => {
|
|
161
|
+
const results = await Promise.all(
|
|
162
|
+
texts.map((text) =>
|
|
163
|
+
env.AI.run('@cf/baai/bge-small-en-v1.5', {
|
|
164
|
+
text,
|
|
165
|
+
}),
|
|
166
|
+
),
|
|
167
|
+
)
|
|
168
|
+
return results.map((r) => r.data[0])
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
export const embedQuery = async (text: string): Promise<number[]> => {
|
|
172
|
+
const result = await env.AI.run('@cf/baai/bge-small-en-v1.5', {
|
|
173
|
+
text,
|
|
174
|
+
})
|
|
175
|
+
return result.data[0]
|
|
176
|
+
}
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
## Using with Voyage AI
|
|
180
|
+
|
|
181
|
+
```typescript
|
|
182
|
+
import { embed, embedMany } from 'ai'
|
|
183
|
+
import { voyage } from 'voyage-ai-provider'
|
|
184
|
+
|
|
185
|
+
export const embedDocs = async (texts: string[]): Promise<number[][]> => {
|
|
186
|
+
const embedResult = await embedMany({
|
|
187
|
+
model: voyage.textEmbeddingModel('voyage-3.5-lite'),
|
|
188
|
+
values: texts,
|
|
189
|
+
providerOptions: {
|
|
190
|
+
voyage: { inputType: 'document' },
|
|
191
|
+
},
|
|
192
|
+
})
|
|
193
|
+
return embedResult.embeddings
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
export const embedQuery = async (text: string): Promise<number[]> => {
|
|
197
|
+
const embedResult = await embed({
|
|
198
|
+
model: voyage.textEmbeddingModel('voyage-3.5-lite'),
|
|
199
|
+
value: text,
|
|
200
|
+
providerOptions: {
|
|
201
|
+
voyage: { inputType: 'query' },
|
|
202
|
+
},
|
|
203
|
+
})
|
|
204
|
+
return embedResult.embedding
|
|
205
|
+
}
|
|
206
|
+
```
|
|
207
|
+
|
|
208
|
+
## License
|
|
209
|
+
|
|
210
|
+
MIT
|
package/dist/embed.js
ADDED
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import { getVectorizedPayload } from 'payloadcms-vectorize';
|
|
2
|
+
/**
|
|
3
|
+
* Store an embedding vector in Cloudflare Vectorize
|
|
4
|
+
* Also creates a Payload document for the metadata
|
|
5
|
+
*/ export default (async (payload, poolName, id, embedding)=>{
|
|
6
|
+
// Get Cloudflare binding from config
|
|
7
|
+
const vectorizeBinding = getVectorizedPayload(payload).getDbAdapterCustom()._vectorizeBinding;
|
|
8
|
+
if (!vectorizeBinding) {
|
|
9
|
+
throw new Error('[@payloadcms-vectorize/cf] Cloudflare Vectorize binding not found');
|
|
10
|
+
}
|
|
11
|
+
try {
|
|
12
|
+
const vector = Array.isArray(embedding) ? embedding : Array.from(embedding);
|
|
13
|
+
// Upsert the vector in Cloudflare Vectorize
|
|
14
|
+
await vectorizeBinding.upsert([
|
|
15
|
+
{
|
|
16
|
+
id,
|
|
17
|
+
values: vector
|
|
18
|
+
}
|
|
19
|
+
]);
|
|
20
|
+
} catch (e) {
|
|
21
|
+
const errorMessage = e.message || e.toString();
|
|
22
|
+
payload.logger.error(`[@payloadcms-vectorize/cf] Failed to store embedding: ${errorMessage}`);
|
|
23
|
+
throw new Error(`[@payloadcms-vectorize/cf] Failed to store embedding: ${errorMessage}`);
|
|
24
|
+
}
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
//# sourceMappingURL=embed.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/embed.ts"],"sourcesContent":["import { Payload } from 'payload'\nimport { getVectorizedPayload } from 'payloadcms-vectorize'\n\n/**\n * Store an embedding vector in Cloudflare Vectorize\n * Also creates a Payload document for the metadata\n */\nexport default async (\n payload: Payload,\n poolName: string,\n id: string,\n embedding: number[] | Float32Array,\n) => {\n // Get Cloudflare binding from config\n const vectorizeBinding = getVectorizedPayload(payload).getDbAdapterCustom()._vectorizeBinding\n if (!vectorizeBinding) {\n throw new Error('[@payloadcms-vectorize/cf] Cloudflare Vectorize binding not found')\n }\n\n try {\n const vector = Array.isArray(embedding) ? embedding : Array.from(embedding)\n\n // Upsert the vector in Cloudflare Vectorize\n await vectorizeBinding.upsert([\n {\n id,\n values: vector,\n },\n ])\n } catch (e) {\n const errorMessage = (e as Error).message || (e as any).toString()\n payload.logger.error(`[@payloadcms-vectorize/cf] Failed to store embedding: ${errorMessage}`)\n throw new Error(`[@payloadcms-vectorize/cf] Failed to store embedding: ${errorMessage}`)\n }\n}\n"],"names":["getVectorizedPayload","payload","poolName","id","embedding","vectorizeBinding","getDbAdapterCustom","_vectorizeBinding","Error","vector","Array","isArray","from","upsert","values","e","errorMessage","message","toString","logger","error"],"mappings":"AACA,SAASA,oBAAoB,QAAQ,uBAAsB;AAE3D;;;CAGC,GACD,eAAe,CAAA,OACbC,SACAC,UACAC,IACAC;IAEA,qCAAqC;IACrC,MAAMC,mBAAmBL,qBAAqBC,SAASK,kBAAkB,GAAGC,iBAAiB;IAC7F,IAAI,CAACF,kBAAkB;QACrB,MAAM,IAAIG,MAAM;IAClB;IAEA,IAAI;QACF,MAAMC,SAASC,MAAMC,OAAO,CAACP,aAAaA,YAAYM,MAAME,IAAI,CAACR;QAEjE,4CAA4C;QAC5C,MAAMC,iBAAiBQ,MAAM,CAAC;YAC5B;gBACEV;gBACAW,QAAQL;YACV;SACD;IACH,EAAE,OAAOM,GAAG;QACV,MAAMC,eAAe,AAACD,EAAYE,OAAO,IAAI,AAACF,EAAUG,QAAQ;QAChEjB,QAAQkB,MAAM,CAACC,KAAK,CAAC,CAAC,sDAAsD,EAAEJ,cAAc;QAC5F,MAAM,IAAIR,MAAM,CAAC,sDAAsD,EAAEQ,cAAc;IACzF;AACF,CAAA,EAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
import embed from './embed';
|
|
2
|
+
import search from './search';
|
|
3
|
+
/**
|
|
4
|
+
* Create a Cloudflare Vectorize integration for payloadcms-vectorize
|
|
5
|
+
*
|
|
6
|
+
* @param options Configuration object with knowledge pools and Vectorize binding
|
|
7
|
+
* @returns Object containing the DbAdapter instance
|
|
8
|
+
*
|
|
9
|
+
* @example
|
|
10
|
+
* ```typescript
|
|
11
|
+
* import { createCloudflareVectorizeIntegration } from '@payloadcms-vectorize/cf'
|
|
12
|
+
*
|
|
13
|
+
* const { adapter } = createCloudflareVectorizeIntegration({
|
|
14
|
+
* config: {
|
|
15
|
+
* default: {
|
|
16
|
+
* dims: 384,
|
|
17
|
+
* },
|
|
18
|
+
* },
|
|
19
|
+
* binding: env.VECTORIZE,
|
|
20
|
+
* })
|
|
21
|
+
* ```
|
|
22
|
+
*/ export const createCloudflareVectorizeIntegration = (options)=>{
|
|
23
|
+
if (!options.binding) {
|
|
24
|
+
throw new Error('[@payloadcms-vectorize/cf] Cloudflare Vectorize binding is required');
|
|
25
|
+
}
|
|
26
|
+
const poolConfig = options.config;
|
|
27
|
+
const adapter = {
|
|
28
|
+
getConfigExtension: ()=>{
|
|
29
|
+
return {
|
|
30
|
+
custom: {
|
|
31
|
+
_cfVectorizeAdapter: true,
|
|
32
|
+
_poolConfigs: poolConfig,
|
|
33
|
+
_vectorizeBinding: options.binding
|
|
34
|
+
}
|
|
35
|
+
};
|
|
36
|
+
},
|
|
37
|
+
search: async (payload, queryEmbedding, poolName, limit, where)=>{
|
|
38
|
+
return search(payload, queryEmbedding, poolName, limit, where);
|
|
39
|
+
},
|
|
40
|
+
storeEmbedding: async (payload, poolName, id, embedding)=>{
|
|
41
|
+
return embed(payload, poolName, id, embedding);
|
|
42
|
+
},
|
|
43
|
+
deleteEmbeddings: async (payload, poolName, sourceCollection, docId)=>{
|
|
44
|
+
// Delete all embeddings for this document from Cloudflare Vectorize
|
|
45
|
+
// First, query to find all matching IDs
|
|
46
|
+
const vectorizeBinding = options.binding;
|
|
47
|
+
const dims = poolConfig[poolName]?.dims || 384;
|
|
48
|
+
try {
|
|
49
|
+
const results = await vectorizeBinding.query(new Array(dims).fill(0), {
|
|
50
|
+
topK: 10000,
|
|
51
|
+
returnMetadata: true,
|
|
52
|
+
where: {
|
|
53
|
+
and: [
|
|
54
|
+
{
|
|
55
|
+
key: 'sourceCollection',
|
|
56
|
+
value: sourceCollection
|
|
57
|
+
},
|
|
58
|
+
{
|
|
59
|
+
key: 'docId',
|
|
60
|
+
value: docId
|
|
61
|
+
}
|
|
62
|
+
]
|
|
63
|
+
}
|
|
64
|
+
});
|
|
65
|
+
const idsToDelete = (results.matches || []).map((match)=>match.id);
|
|
66
|
+
if (idsToDelete.length > 0) {
|
|
67
|
+
await vectorizeBinding.delete(idsToDelete);
|
|
68
|
+
}
|
|
69
|
+
} catch (error) {
|
|
70
|
+
const errorMessage = error.message || error.toString();
|
|
71
|
+
payload.logger.error(`[@payloadcms-vectorize/cf] Failed to delete embeddings: ${errorMessage}`);
|
|
72
|
+
throw new Error(`[@payloadcms-vectorize/cf] Failed to delete embeddings: ${errorMessage}`);
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
};
|
|
76
|
+
return {
|
|
77
|
+
adapter
|
|
78
|
+
};
|
|
79
|
+
};
|
|
80
|
+
|
|
81
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/index.ts"],"sourcesContent":["import type { DbAdapter } from 'payloadcms-vectorize'\nimport type { CloudflareVectorizeBinding, KnowledgePoolsConfig } from './types'\nimport embed from './embed'\nimport search from './search'\n\n/**\n * Configuration for Cloudflare Vectorize integration\n */\ninterface CloudflareVectorizeConfig {\n /** Knowledge pools configuration with their dimensions */\n config: KnowledgePoolsConfig\n /** Cloudflare Vectorize binding for vector storage */\n binding: CloudflareVectorizeBinding\n}\n\n/**\n * Create a Cloudflare Vectorize integration for payloadcms-vectorize\n *\n * @param options Configuration object with knowledge pools and Vectorize binding\n * @returns Object containing the DbAdapter instance\n *\n * @example\n * ```typescript\n * import { createCloudflareVectorizeIntegration } from '@payloadcms-vectorize/cf'\n *\n * const { adapter } = createCloudflareVectorizeIntegration({\n * config: {\n * default: {\n * dims: 384,\n * },\n * },\n * binding: env.VECTORIZE,\n * })\n * ```\n */\nexport const createCloudflareVectorizeIntegration = (\n options: CloudflareVectorizeConfig,\n): { adapter: DbAdapter } => {\n if (!options.binding) {\n throw new Error('[@payloadcms-vectorize/cf] Cloudflare Vectorize binding is required')\n }\n\n const poolConfig = options.config\n\n const adapter: DbAdapter = {\n getConfigExtension: () => {\n return {\n custom: {\n _cfVectorizeAdapter: true,\n _poolConfigs: poolConfig,\n _vectorizeBinding: options.binding,\n },\n }\n },\n\n search: async (payload, queryEmbedding, poolName, limit, where) => {\n return search(payload, queryEmbedding, poolName, limit, where)\n },\n\n storeEmbedding: async (payload, poolName, id, embedding) => {\n return embed(payload, poolName, id, embedding)\n },\n\n deleteEmbeddings: async (payload, poolName, sourceCollection, docId) => {\n // Delete all embeddings for this document from Cloudflare Vectorize\n // First, query to find all matching IDs\n const vectorizeBinding = options.binding\n const dims = poolConfig[poolName]?.dims || 384\n try {\n const results = await vectorizeBinding.query(new Array(dims).fill(0), {\n topK: 10000,\n returnMetadata: true,\n where: {\n and: [\n { key: 'sourceCollection', value: sourceCollection },\n { key: 'docId', value: docId },\n ],\n },\n })\n\n const idsToDelete = (results.matches || []).map((match: any) => match.id)\n\n if (idsToDelete.length > 0) {\n await vectorizeBinding.delete(idsToDelete)\n }\n } catch (error) {\n const errorMessage = (error as Error).message || (error as any).toString()\n payload.logger.error(\n `[@payloadcms-vectorize/cf] Failed to delete embeddings: ${errorMessage}`,\n )\n throw new Error(`[@payloadcms-vectorize/cf] Failed to delete embeddings: ${errorMessage}`)\n }\n },\n }\n\n return { adapter }\n}\n\nexport type { CloudflareVectorizeBinding, KnowledgePoolsConfig }\nexport type { KnowledgePoolsConfig as KnowledgePoolConfig }\n"],"names":["embed","search","createCloudflareVectorizeIntegration","options","binding","Error","poolConfig","config","adapter","getConfigExtension","custom","_cfVectorizeAdapter","_poolConfigs","_vectorizeBinding","payload","queryEmbedding","poolName","limit","where","storeEmbedding","id","embedding","deleteEmbeddings","sourceCollection","docId","vectorizeBinding","dims","results","query","Array","fill","topK","returnMetadata","and","key","value","idsToDelete","matches","map","match","length","delete","error","errorMessage","message","toString","logger"],"mappings":"AAEA,OAAOA,WAAW,UAAS;AAC3B,OAAOC,YAAY,WAAU;AAY7B;;;;;;;;;;;;;;;;;;;CAmBC,GACD,OAAO,MAAMC,uCAAuC,CAClDC;IAEA,IAAI,CAACA,QAAQC,OAAO,EAAE;QACpB,MAAM,IAAIC,MAAM;IAClB;IAEA,MAAMC,aAAaH,QAAQI,MAAM;IAEjC,MAAMC,UAAqB;QACzBC,oBAAoB;YAClB,OAAO;gBACLC,QAAQ;oBACNC,qBAAqB;oBACrBC,cAAcN;oBACdO,mBAAmBV,QAAQC,OAAO;gBACpC;YACF;QACF;QAEAH,QAAQ,OAAOa,SAASC,gBAAgBC,UAAUC,OAAOC;YACvD,OAAOjB,OAAOa,SAASC,gBAAgBC,UAAUC,OAAOC;QAC1D;QAEAC,gBAAgB,OAAOL,SAASE,UAAUI,IAAIC;YAC5C,OAAOrB,MAAMc,SAASE,UAAUI,IAAIC;QACtC;QAEAC,kBAAkB,OAAOR,SAASE,UAAUO,kBAAkBC;YAC5D,oEAAoE;YACpE,wCAAwC;YACxC,MAAMC,mBAAmBtB,QAAQC,OAAO;YACxC,MAAMsB,OAAOpB,UAAU,CAACU,SAAS,EAAEU,QAAQ;YAC3C,IAAI;gBACF,MAAMC,UAAU,MAAMF,iBAAiBG,KAAK,CAAC,IAAIC,MAAMH,MAAMI,IAAI,CAAC,IAAI;oBACpEC,MAAM;oBACNC,gBAAgB;oBAChBd,OAAO;wBACLe,KAAK;4BACH;gCAAEC,KAAK;gCAAoBC,OAAOZ;4BAAiB;4BACnD;gCAAEW,KAAK;gCAASC,OAAOX;4BAAM;yBAC9B;oBACH;gBACF;gBAEA,MAAMY,cAAc,AAACT,CAAAA,QAAQU,OAAO,IAAI,EAAE,AAAD,EAAGC,GAAG,CAAC,CAACC,QAAeA,MAAMnB,EAAE;gBAExE,IAAIgB,YAAYI,MAAM,GAAG,GAAG;oBAC1B,MAAMf,iBAAiBgB,MAAM,CAACL;gBAChC;YACF,EAAE,OAAOM,OAAO;gBACd,MAAMC,eAAe,AAACD,MAAgBE,OAAO,IAAI,AAACF,MAAcG,QAAQ;gBACxE/B,QAAQgC,MAAM,CAACJ,KAAK,CAClB,CAAC,wDAAwD,EAAEC,cAAc;gBAE3E,MAAM,IAAItC,MAAM,CAAC,wDAAwD,EAAEsC,cAAc;YAC3F;QACF;IACF;IAEA,OAAO;QAAEnC;IAAQ;AACnB,EAAC"}
|
package/dist/search.js
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { getVectorizedPayload } from 'payloadcms-vectorize';
|
|
2
|
+
/**
|
|
3
|
+
* Search for similar vectors in Cloudflare Vectorize
|
|
4
|
+
*/ export default (async (payload, queryEmbedding, poolName, limit = 10, where)=>{
|
|
5
|
+
// Get Cloudflare binding from config
|
|
6
|
+
const vectorizeBinding = getVectorizedPayload(payload).getDbAdapterCustom()._vectorizeBinding;
|
|
7
|
+
if (!vectorizeBinding) {
|
|
8
|
+
throw new Error('[@payloadcms-vectorize/cf] Cloudflare Vectorize binding not found');
|
|
9
|
+
}
|
|
10
|
+
try {
|
|
11
|
+
// Get collection config
|
|
12
|
+
const collectionConfig = payload.collections[poolName]?.config;
|
|
13
|
+
if (!collectionConfig) {
|
|
14
|
+
throw new Error(`Collection ${poolName} not found`);
|
|
15
|
+
}
|
|
16
|
+
// Query Cloudflare Vectorize
|
|
17
|
+
// The query returns the top-k most similar vectors
|
|
18
|
+
const results = await vectorizeBinding.query(queryEmbedding, {
|
|
19
|
+
topK: limit,
|
|
20
|
+
returnMetadata: true
|
|
21
|
+
});
|
|
22
|
+
if (!results.matches) {
|
|
23
|
+
return [];
|
|
24
|
+
}
|
|
25
|
+
// Fetch full documents from Payload for metadata
|
|
26
|
+
const searchResults = [];
|
|
27
|
+
for (const match of results.matches){
|
|
28
|
+
try {
|
|
29
|
+
const doc = await payload.findByID({
|
|
30
|
+
collection: poolName,
|
|
31
|
+
id: match.id
|
|
32
|
+
});
|
|
33
|
+
if (doc && (!where || matchesWhere(doc, where))) {
|
|
34
|
+
// Extract fields excluding internal ones
|
|
35
|
+
const { id: _id, createdAt: _createdAt, updatedAt: _updatedAt, ...docFields } = doc;
|
|
36
|
+
searchResults.push({
|
|
37
|
+
id: match.id,
|
|
38
|
+
score: match.score || 0,
|
|
39
|
+
...docFields
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
} catch (_e) {
|
|
43
|
+
// Document not found or error fetching, skip
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
return searchResults;
|
|
47
|
+
} catch (e) {
|
|
48
|
+
const errorMessage = e.message || e.toString();
|
|
49
|
+
payload.logger.error(`[@payloadcms-vectorize/cf] Search failed: ${errorMessage}`);
|
|
50
|
+
throw new Error(`[@payloadcms-vectorize/cf] Search failed: ${errorMessage}`);
|
|
51
|
+
}
|
|
52
|
+
});
|
|
53
|
+
/**
|
|
54
|
+
* Simple WHERE clause matcher for basic filtering
|
|
55
|
+
* Supports: equals, in, exists, and, or
|
|
56
|
+
*/ function matchesWhere(doc, where) {
|
|
57
|
+
if (!where || Object.keys(where).length === 0) return true;
|
|
58
|
+
// Handle 'and' operator
|
|
59
|
+
if ('and' in where && Array.isArray(where.and)) {
|
|
60
|
+
return where.and.every((clause)=>matchesWhere(doc, clause));
|
|
61
|
+
}
|
|
62
|
+
// Handle 'or' operator
|
|
63
|
+
if ('or' in where && Array.isArray(where.or)) {
|
|
64
|
+
return where.or.some((clause)=>matchesWhere(doc, clause));
|
|
65
|
+
}
|
|
66
|
+
// Handle field-level conditions
|
|
67
|
+
for (const [field, condition] of Object.entries(where)){
|
|
68
|
+
if (field === 'and' || field === 'or') continue;
|
|
69
|
+
const value = doc[field];
|
|
70
|
+
if (typeof condition === 'object' && condition !== null) {
|
|
71
|
+
if ('equals' in condition && value !== condition.equals) {
|
|
72
|
+
return false;
|
|
73
|
+
}
|
|
74
|
+
if ('in' in condition && Array.isArray(condition.in) && !condition.in.includes(value)) {
|
|
75
|
+
return false;
|
|
76
|
+
}
|
|
77
|
+
if ('exists' in condition) {
|
|
78
|
+
const exists = value !== undefined && value !== null;
|
|
79
|
+
if (condition.exists !== exists) return false;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
return true;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
//# sourceMappingURL=search.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/search.ts"],"sourcesContent":["import { BasePayload, Where } from 'payload'\nimport { KnowledgePoolName, VectorSearchResult, getVectorizedPayload } from 'payloadcms-vectorize'\n\n/**\n * Search for similar vectors in Cloudflare Vectorize\n */\nexport default async (\n payload: BasePayload,\n queryEmbedding: number[],\n poolName: KnowledgePoolName,\n limit: number = 10,\n where?: Where,\n): Promise<Array<VectorSearchResult>> => {\n // Get Cloudflare binding from config\n const vectorizeBinding = getVectorizedPayload(payload).getDbAdapterCustom()._vectorizeBinding\n if (!vectorizeBinding) {\n throw new Error('[@payloadcms-vectorize/cf] Cloudflare Vectorize binding not found')\n }\n\n try {\n // Get collection config\n const collectionConfig = payload.collections[poolName]?.config\n if (!collectionConfig) {\n throw new Error(`Collection ${poolName} not found`)\n }\n\n // Query Cloudflare Vectorize\n // The query returns the top-k most similar vectors\n const results = await vectorizeBinding.query(queryEmbedding, {\n topK: limit,\n returnMetadata: true,\n })\n\n if (!results.matches) {\n return []\n }\n\n // Fetch full documents from Payload for metadata\n const searchResults: VectorSearchResult[] = []\n\n for (const match of results.matches) {\n try {\n const doc = await payload.findByID({\n collection: poolName as any,\n id: match.id,\n })\n\n if (doc && (!where || matchesWhere(doc, where))) {\n // Extract fields excluding internal ones\n const { id: _id, createdAt: _createdAt, updatedAt: _updatedAt, ...docFields } = doc as any\n\n searchResults.push({\n id: match.id,\n score: match.score || 0,\n ...docFields, // Includes sourceCollection, docId, chunkText, embeddingVersion, extension fields\n })\n }\n } catch (_e) {\n // Document not found or error fetching, skip\n }\n }\n\n return searchResults\n } catch (e) {\n const errorMessage = (e as Error).message || (e as any).toString()\n payload.logger.error(`[@payloadcms-vectorize/cf] Search failed: ${errorMessage}`)\n throw new Error(`[@payloadcms-vectorize/cf] Search failed: ${errorMessage}`)\n }\n}\n\n/**\n * Simple WHERE clause matcher for basic filtering\n * Supports: equals, in, exists, and, or\n */\nfunction matchesWhere(doc: Record<string, any>, where: Where): boolean {\n if (!where || Object.keys(where).length === 0) return true\n\n // Handle 'and' operator\n if ('and' in where && Array.isArray(where.and)) {\n return where.and.every((clause: Where) => matchesWhere(doc, clause))\n }\n\n // Handle 'or' operator\n if ('or' in where && Array.isArray(where.or)) {\n return where.or.some((clause: Where) => matchesWhere(doc, clause))\n }\n\n // Handle field-level conditions\n for (const [field, condition] of Object.entries(where)) {\n if (field === 'and' || field === 'or') continue\n\n const value = doc[field]\n\n if (typeof condition === 'object' && condition !== null) {\n if ('equals' in condition && value !== condition.equals) {\n return false\n }\n if ('in' in condition && Array.isArray(condition.in) && !condition.in.includes(value)) {\n return false\n }\n if ('exists' in condition) {\n const exists = value !== undefined && value !== null\n if (condition.exists !== exists) return false\n }\n }\n }\n\n return true\n}\n"],"names":["getVectorizedPayload","payload","queryEmbedding","poolName","limit","where","vectorizeBinding","getDbAdapterCustom","_vectorizeBinding","Error","collectionConfig","collections","config","results","query","topK","returnMetadata","matches","searchResults","match","doc","findByID","collection","id","matchesWhere","_id","createdAt","_createdAt","updatedAt","_updatedAt","docFields","push","score","_e","e","errorMessage","message","toString","logger","error","Object","keys","length","Array","isArray","and","every","clause","or","some","field","condition","entries","value","equals","in","includes","exists","undefined"],"mappings":"AACA,SAAgDA,oBAAoB,QAAQ,uBAAsB;AAElG;;CAEC,GACD,eAAe,CAAA,OACbC,SACAC,gBACAC,UACAC,QAAgB,EAAE,EAClBC;IAEA,qCAAqC;IACrC,MAAMC,mBAAmBN,qBAAqBC,SAASM,kBAAkB,GAAGC,iBAAiB;IAC7F,IAAI,CAACF,kBAAkB;QACrB,MAAM,IAAIG,MAAM;IAClB;IAEA,IAAI;QACF,wBAAwB;QACxB,MAAMC,mBAAmBT,QAAQU,WAAW,CAACR,SAAS,EAAES;QACxD,IAAI,CAACF,kBAAkB;YACrB,MAAM,IAAID,MAAM,CAAC,WAAW,EAAEN,SAAS,UAAU,CAAC;QACpD;QAEA,6BAA6B;QAC7B,mDAAmD;QACnD,MAAMU,UAAU,MAAMP,iBAAiBQ,KAAK,CAACZ,gBAAgB;YAC3Da,MAAMX;YACNY,gBAAgB;QAClB;QAEA,IAAI,CAACH,QAAQI,OAAO,EAAE;YACpB,OAAO,EAAE;QACX;QAEA,iDAAiD;QACjD,MAAMC,gBAAsC,EAAE;QAE9C,KAAK,MAAMC,SAASN,QAAQI,OAAO,CAAE;YACnC,IAAI;gBACF,MAAMG,MAAM,MAAMnB,QAAQoB,QAAQ,CAAC;oBACjCC,YAAYnB;oBACZoB,IAAIJ,MAAMI,EAAE;gBACd;gBAEA,IAAIH,OAAQ,CAAA,CAACf,SAASmB,aAAaJ,KAAKf,MAAK,GAAI;oBAC/C,yCAAyC;oBACzC,MAAM,EAAEkB,IAAIE,GAAG,EAAEC,WAAWC,UAAU,EAAEC,WAAWC,UAAU,EAAE,GAAGC,WAAW,GAAGV;oBAEhFF,cAAca,IAAI,CAAC;wBACjBR,IAAIJ,MAAMI,EAAE;wBACZS,OAAOb,MAAMa,KAAK,IAAI;wBACtB,GAAGF,SAAS;oBACd;gBACF;YACF,EAAE,OAAOG,IAAI;YACX,6CAA6C;YAC/C;QACF;QAEA,OAAOf;IACT,EAAE,OAAOgB,GAAG;QACV,MAAMC,eAAe,AAACD,EAAYE,OAAO,IAAI,AAACF,EAAUG,QAAQ;QAChEpC,QAAQqC,MAAM,CAACC,KAAK,CAAC,CAAC,0CAA0C,EAAEJ,cAAc;QAChF,MAAM,IAAI1B,MAAM,CAAC,0CAA0C,EAAE0B,cAAc;IAC7E;AACF,CAAA,EAAC;AAED;;;CAGC,GACD,SAASX,aAAaJ,GAAwB,EAAEf,KAAY;IAC1D,IAAI,CAACA,SAASmC,OAAOC,IAAI,CAACpC,OAAOqC,MAAM,KAAK,GAAG,OAAO;IAEtD,wBAAwB;IACxB,IAAI,SAASrC,SAASsC,MAAMC,OAAO,CAACvC,MAAMwC,GAAG,GAAG;QAC9C,OAAOxC,MAAMwC,GAAG,CAACC,KAAK,CAAC,CAACC,SAAkBvB,aAAaJ,KAAK2B;IAC9D;IAEA,uBAAuB;IACvB,IAAI,QAAQ1C,SAASsC,MAAMC,OAAO,CAACvC,MAAM2C,EAAE,GAAG;QAC5C,OAAO3C,MAAM2C,EAAE,CAACC,IAAI,CAAC,CAACF,SAAkBvB,aAAaJ,KAAK2B;IAC5D;IAEA,gCAAgC;IAChC,KAAK,MAAM,CAACG,OAAOC,UAAU,IAAIX,OAAOY,OAAO,CAAC/C,OAAQ;QACtD,IAAI6C,UAAU,SAASA,UAAU,MAAM;QAEvC,MAAMG,QAAQjC,GAAG,CAAC8B,MAAM;QAExB,IAAI,OAAOC,cAAc,YAAYA,cAAc,MAAM;YACvD,IAAI,YAAYA,aAAaE,UAAUF,UAAUG,MAAM,EAAE;gBACvD,OAAO;YACT;YACA,IAAI,QAAQH,aAAaR,MAAMC,OAAO,CAACO,UAAUI,EAAE,KAAK,CAACJ,UAAUI,EAAE,CAACC,QAAQ,CAACH,QAAQ;gBACrF,OAAO;YACT;YACA,IAAI,YAAYF,WAAW;gBACzB,MAAMM,SAASJ,UAAUK,aAAaL,UAAU;gBAChD,IAAIF,UAAUM,MAAM,KAAKA,QAAQ,OAAO;YAC1C;QACF;IACF;IAEA,OAAO;AACT"}
|
package/dist/types.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../src/types.ts"],"sourcesContent":["import { BasePayload } from 'payload'\n\n/**\n * Configuration for a knowledge pool in Cloudflare Vectorize\n */\nexport interface CloudflareVectorizePoolConfig {\n /** Vector dimensions for this pool (must match embedding model output) */\n dims: number\n}\n\n/**\n * All knowledge pools configuration for Cloudflare Vectorize\n */\nexport type KnowledgePoolsConfig = Record<string, CloudflareVectorizePoolConfig>\n\n/**\n * Cloudflare Vectorize binding for vector storage\n */\nexport type CloudflareVectorizeBinding = any\n"],"names":[],"mappings":"AAeA;;CAEC,GACD,WAA4C"}
|
package/package.json
ADDED
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@payloadcms-vectorize/cf",
|
|
3
|
+
"version": "0.6.0-beta",
|
|
4
|
+
"description": "Cloudflare Vectorize adapter for payloadcms-vectorize",
|
|
5
|
+
"license": "MIT",
|
|
6
|
+
"type": "module",
|
|
7
|
+
"files": [
|
|
8
|
+
"dist"
|
|
9
|
+
],
|
|
10
|
+
"main": "./dist/index.js",
|
|
11
|
+
"types": "./dist/index.d.ts",
|
|
12
|
+
"peerDependencies": {
|
|
13
|
+
"payload": ">=3.0.0 <4.0.0",
|
|
14
|
+
"payloadcms-vectorize": ">=0.6.0-beta <1.0.0"
|
|
15
|
+
},
|
|
16
|
+
"devDependencies": {
|
|
17
|
+
"payloadcms-vectorize": "0.6.0-beta"
|
|
18
|
+
},
|
|
19
|
+
"engines": {
|
|
20
|
+
"node": "^18.20.2 || >=20.9.0",
|
|
21
|
+
"pnpm": "^9 || ^10"
|
|
22
|
+
}
|
|
23
|
+
}
|