@payloadcms-vectorize/cf 0.6.0-beta.2 → 0.6.0-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/dist/collections/cfMappings.js +46 -0
- package/dist/collections/cfMappings.js.map +1 -0
- package/dist/embed.js +15 -9
- package/dist/embed.js.map +1 -1
- package/dist/index.js +63 -22
- package/dist/index.js.map +1 -1
- package/dist/search.js +39 -32
- package/dist/search.js.map +1 -1
- package/dist/types.js +10 -2
- package/dist/types.js.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -205,6 +205,10 @@ export const embedQuery = async (text: string): Promise<number[]> => {
|
|
|
205
205
|
}
|
|
206
206
|
```
|
|
207
207
|
|
|
208
|
+
## Known Limitations
|
|
209
|
+
|
|
210
|
+
- **Search `limit` with `where` filtering:** When a `where` clause is provided, filtering is applied after fetching results from Cloudflare Vectorize. This means you may receive fewer results than the requested `limit` even when more matching vectors exist.
|
|
211
|
+
|
|
208
212
|
## License
|
|
209
213
|
|
|
210
214
|
MIT
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
export const CF_MAPPINGS_SLUG = 'vector-cf-mappings';
|
|
2
|
+
// This collection maps Cloudflare Vectorize vector IDs to source documents,
|
|
3
|
+
// so we can find and delete vectors when the source document is deleted.
|
|
4
|
+
const CFMappingsCollection = {
|
|
5
|
+
slug: CF_MAPPINGS_SLUG,
|
|
6
|
+
admin: {
|
|
7
|
+
hidden: true,
|
|
8
|
+
description: 'Maps Cloudflare Vectorize vector IDs to source documents. Managed by the CF adapter.'
|
|
9
|
+
},
|
|
10
|
+
access: {
|
|
11
|
+
read: ()=>true,
|
|
12
|
+
create: ({ req })=>req?.payloadAPI === 'local',
|
|
13
|
+
update: ({ req })=>req?.payloadAPI === 'local',
|
|
14
|
+
delete: ({ req })=>req?.payloadAPI === 'local'
|
|
15
|
+
},
|
|
16
|
+
fields: [
|
|
17
|
+
{
|
|
18
|
+
name: 'vectorId',
|
|
19
|
+
type: 'text',
|
|
20
|
+
required: true,
|
|
21
|
+
index: true
|
|
22
|
+
},
|
|
23
|
+
{
|
|
24
|
+
name: 'poolName',
|
|
25
|
+
type: 'text',
|
|
26
|
+
required: true,
|
|
27
|
+
index: true
|
|
28
|
+
},
|
|
29
|
+
{
|
|
30
|
+
name: 'sourceCollection',
|
|
31
|
+
type: 'text',
|
|
32
|
+
required: true,
|
|
33
|
+
index: true
|
|
34
|
+
},
|
|
35
|
+
{
|
|
36
|
+
name: 'docId',
|
|
37
|
+
type: 'text',
|
|
38
|
+
required: true,
|
|
39
|
+
index: true
|
|
40
|
+
}
|
|
41
|
+
],
|
|
42
|
+
timestamps: true
|
|
43
|
+
};
|
|
44
|
+
export default CFMappingsCollection;
|
|
45
|
+
|
|
46
|
+
//# sourceMappingURL=cfMappings.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"sources":["../../src/collections/cfMappings.ts"],"sourcesContent":["import type { CollectionConfig } from 'payload'\n\nexport const CF_MAPPINGS_SLUG = 'vector-cf-mappings'\n\n// This collection maps Cloudflare Vectorize vector IDs to source documents,\n// so we can find and delete vectors when the source document is deleted.\nconst CFMappingsCollection: CollectionConfig = {\n slug: CF_MAPPINGS_SLUG,\n admin: {\n hidden: true,\n description:\n 'Maps Cloudflare Vectorize vector IDs to source documents. Managed by the CF adapter.',\n },\n access: {\n read: () => true,\n create: ({ req }) => req?.payloadAPI === 'local',\n update: ({ req }) => req?.payloadAPI === 'local',\n delete: ({ req }) => req?.payloadAPI === 'local',\n },\n fields: [\n {\n name: 'vectorId',\n type: 'text',\n required: true,\n index: true,\n },\n {\n name: 'poolName',\n type: 'text',\n required: true,\n index: true,\n },\n {\n name: 'sourceCollection',\n type: 'text',\n required: true,\n index: true,\n },\n {\n name: 'docId',\n type: 'text',\n required: true,\n index: true,\n },\n ],\n timestamps: true,\n}\n\nexport default CFMappingsCollection\n"],"names":["CF_MAPPINGS_SLUG","CFMappingsCollection","slug","admin","hidden","description","access","read","create","req","payloadAPI","update","delete","fields","name","type","required","index","timestamps"],"mappings":"AAEA,OAAO,MAAMA,mBAAmB,qBAAoB;AAEpD,4EAA4E;AAC5E,yEAAyE;AACzE,MAAMC,uBAAyC;IAC7CC,MAAMF;IACNG,OAAO;QACLC,QAAQ;QACRC,aACE;IACJ;IACAC,QAAQ;QACNC,MAAM,IAAM;QACZC,QAAQ,CAAC,EAAEC,GAAG,EAAE,GAAKA,KAAKC,eAAe;QACzCC,QAAQ,CAAC,EAAEF,GAAG,EAAE,GAAKA,KAAKC,eAAe;QACzCE,QAAQ,CAAC,EAAEH,GAAG,EAAE,GAAKA,KAAKC,eAAe;IAC3C;IACAG,QAAQ;QACN;YACEC,MAAM;YACNC,MAAM;YACNC,UAAU;YACVC,OAAO;QACT;QACA;YACEH,MAAM;YACNC,MAAM;YACNC,UAAU;YACVC,OAAO;QACT;QACA;YACEH,MAAM;YACNC,MAAM;YACNC,UAAU;YACVC,OAAO;QACT;QACA;YACEH,MAAM;YACNC,MAAM;YACNC,UAAU;YACVC,OAAO;QACT;KACD;IACDC,YAAY;AACd;AAEA,eAAejB,qBAAoB"}
|
package/dist/embed.js
CHANGED
|
@@ -1,13 +1,9 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { getVectorizeBinding } from './types.js';
|
|
2
|
+
import { CF_MAPPINGS_SLUG } from './collections/cfMappings.js';
|
|
2
3
|
/**
|
|
3
4
|
* Store an embedding vector in Cloudflare Vectorize
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
// Get Cloudflare binding from config
|
|
7
|
-
const vectorizeBinding = getVectorizedPayload(payload)?.getDbAdapterCustom()?._vectorizeBinding;
|
|
8
|
-
if (!vectorizeBinding) {
|
|
9
|
-
throw new Error('[@payloadcms-vectorize/cf] Cloudflare Vectorize binding not found');
|
|
10
|
-
}
|
|
5
|
+
*/ export default (async (payload, poolName, sourceCollection, sourceDocId, id, embedding)=>{
|
|
6
|
+
const vectorizeBinding = getVectorizeBinding(payload);
|
|
11
7
|
try {
|
|
12
8
|
const vector = Array.isArray(embedding) ? embedding : Array.from(embedding);
|
|
13
9
|
// Upsert the vector in Cloudflare Vectorize
|
|
@@ -17,8 +13,18 @@ import { getVectorizedPayload } from 'payloadcms-vectorize';
|
|
|
17
13
|
values: vector
|
|
18
14
|
}
|
|
19
15
|
]);
|
|
16
|
+
// Create a mapping row so we can find this vector during deletion
|
|
17
|
+
await payload.create({
|
|
18
|
+
collection: CF_MAPPINGS_SLUG,
|
|
19
|
+
data: {
|
|
20
|
+
vectorId: id,
|
|
21
|
+
poolName,
|
|
22
|
+
sourceCollection,
|
|
23
|
+
docId: sourceDocId
|
|
24
|
+
}
|
|
25
|
+
});
|
|
20
26
|
} catch (e) {
|
|
21
|
-
const errorMessage = e.message
|
|
27
|
+
const errorMessage = e instanceof Error ? e.message : String(e);
|
|
22
28
|
payload.logger.error(`[@payloadcms-vectorize/cf] Failed to store embedding: ${errorMessage}`);
|
|
23
29
|
throw new Error(`[@payloadcms-vectorize/cf] Failed to store embedding: ${errorMessage}`);
|
|
24
30
|
}
|
package/dist/embed.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/embed.ts"],"sourcesContent":["import { Payload } from 'payload'\nimport {
|
|
1
|
+
{"version":3,"sources":["../src/embed.ts"],"sourcesContent":["import { CollectionSlug, Payload } from 'payload'\nimport { getVectorizeBinding } from './types.js'\nimport { CF_MAPPINGS_SLUG } from './collections/cfMappings.js'\n\n/**\n * Store an embedding vector in Cloudflare Vectorize\n */\nexport default async (\n payload: Payload,\n poolName: string,\n sourceCollection: string,\n sourceDocId: string,\n id: string,\n embedding: number[] | Float32Array,\n) => {\n const vectorizeBinding = getVectorizeBinding(payload)\n\n try {\n const vector = Array.isArray(embedding) ? embedding : Array.from(embedding)\n\n // Upsert the vector in Cloudflare Vectorize\n await vectorizeBinding.upsert([\n {\n id,\n values: vector,\n },\n ])\n\n // Create a mapping row so we can find this vector during deletion\n await payload.create({\n collection: CF_MAPPINGS_SLUG as CollectionSlug,\n data: {\n vectorId: id,\n poolName,\n sourceCollection,\n docId: sourceDocId,\n },\n })\n } catch (e) {\n const errorMessage = e instanceof Error ? e.message : String(e)\n payload.logger.error(`[@payloadcms-vectorize/cf] Failed to store embedding: ${errorMessage}`)\n throw new Error(`[@payloadcms-vectorize/cf] Failed to store embedding: ${errorMessage}`)\n }\n}\n"],"names":["getVectorizeBinding","CF_MAPPINGS_SLUG","payload","poolName","sourceCollection","sourceDocId","id","embedding","vectorizeBinding","vector","Array","isArray","from","upsert","values","create","collection","data","vectorId","docId","e","errorMessage","Error","message","String","logger","error"],"mappings":"AACA,SAASA,mBAAmB,QAAQ,aAAY;AAChD,SAASC,gBAAgB,QAAQ,8BAA6B;AAE9D;;CAEC,GACD,eAAe,CAAA,OACbC,SACAC,UACAC,kBACAC,aACAC,IACAC;IAEA,MAAMC,mBAAmBR,oBAAoBE;IAE7C,IAAI;QACF,MAAMO,SAASC,MAAMC,OAAO,CAACJ,aAAaA,YAAYG,MAAME,IAAI,CAACL;QAEjE,4CAA4C;QAC5C,MAAMC,iBAAiBK,MAAM,CAAC;YAC5B;gBACEP;gBACAQ,QAAQL;YACV;SACD;QAED,kEAAkE;QAClE,MAAMP,QAAQa,MAAM,CAAC;YACnBC,YAAYf;YACZgB,MAAM;gBACJC,UAAUZ;gBACVH;gBACAC;gBACAe,OAAOd;YACT;QACF;IACF,EAAE,OAAOe,GAAG;QACV,MAAMC,eAAeD,aAAaE,QAAQF,EAAEG,OAAO,GAAGC,OAAOJ;QAC7DlB,QAAQuB,MAAM,CAACC,KAAK,CAAC,CAAC,sDAAsD,EAAEL,cAAc;QAC5F,MAAM,IAAIC,MAAM,CAAC,sDAAsD,EAAED,cAAc;IACzF;AACF,CAAA,EAAC"}
|
package/dist/index.js
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { getVectorizeBinding } from './types.js';
|
|
2
|
+
import cfMappingsCollection, { CF_MAPPINGS_SLUG } from './collections/cfMappings.js';
|
|
1
3
|
import embed from './embed.js';
|
|
2
4
|
import search from './search.js';
|
|
3
5
|
/**
|
|
@@ -27,6 +29,9 @@ import search from './search.js';
|
|
|
27
29
|
const adapter = {
|
|
28
30
|
getConfigExtension: ()=>{
|
|
29
31
|
return {
|
|
32
|
+
collections: {
|
|
33
|
+
[CF_MAPPINGS_SLUG]: cfMappingsCollection
|
|
34
|
+
},
|
|
30
35
|
custom: {
|
|
31
36
|
_cfVectorizeAdapter: true,
|
|
32
37
|
_poolConfigs: poolConfig,
|
|
@@ -34,40 +39,75 @@ import search from './search.js';
|
|
|
34
39
|
}
|
|
35
40
|
};
|
|
36
41
|
},
|
|
37
|
-
search
|
|
38
|
-
|
|
39
|
-
},
|
|
40
|
-
storeEmbedding: async (payload, poolName, id, embedding)=>{
|
|
41
|
-
return embed(payload, poolName, id, embedding);
|
|
42
|
-
},
|
|
42
|
+
search,
|
|
43
|
+
storeEmbedding: embed,
|
|
43
44
|
deleteEmbeddings: async (payload, poolName, sourceCollection, docId)=>{
|
|
44
|
-
|
|
45
|
-
// First, query to find all matching IDs
|
|
46
|
-
const vectorizeBinding = options.binding;
|
|
47
|
-
const dims = poolConfig[poolName]?.dims || 384;
|
|
45
|
+
const vectorizeBinding = getVectorizeBinding(payload);
|
|
48
46
|
try {
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
47
|
+
// Paginate through all mapping rows for this document+pool
|
|
48
|
+
const allVectorIds = [];
|
|
49
|
+
let page = 1;
|
|
50
|
+
let hasNextPage = true;
|
|
51
|
+
while(hasNextPage){
|
|
52
|
+
const mappings = await payload.find({
|
|
53
|
+
collection: CF_MAPPINGS_SLUG,
|
|
54
|
+
where: {
|
|
55
|
+
and: [
|
|
56
|
+
{
|
|
57
|
+
poolName: {
|
|
58
|
+
equals: poolName
|
|
59
|
+
}
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
sourceCollection: {
|
|
63
|
+
equals: sourceCollection
|
|
64
|
+
}
|
|
65
|
+
},
|
|
66
|
+
{
|
|
67
|
+
docId: {
|
|
68
|
+
equals: docId
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
]
|
|
72
|
+
},
|
|
73
|
+
page
|
|
74
|
+
});
|
|
75
|
+
for (const mapping of mappings.docs){
|
|
76
|
+
allVectorIds.push(mapping.vectorId);
|
|
77
|
+
}
|
|
78
|
+
hasNextPage = mappings.hasNextPage;
|
|
79
|
+
page++;
|
|
80
|
+
}
|
|
81
|
+
if (allVectorIds.length === 0) {
|
|
82
|
+
return;
|
|
83
|
+
}
|
|
84
|
+
// Delete vectors from Cloudflare Vectorize
|
|
85
|
+
await vectorizeBinding.deleteByIds(allVectorIds);
|
|
86
|
+
// Delete mapping rows
|
|
87
|
+
await payload.delete({
|
|
88
|
+
collection: CF_MAPPINGS_SLUG,
|
|
52
89
|
where: {
|
|
53
90
|
and: [
|
|
54
91
|
{
|
|
55
|
-
|
|
56
|
-
|
|
92
|
+
poolName: {
|
|
93
|
+
equals: poolName
|
|
94
|
+
}
|
|
95
|
+
},
|
|
96
|
+
{
|
|
97
|
+
sourceCollection: {
|
|
98
|
+
equals: sourceCollection
|
|
99
|
+
}
|
|
57
100
|
},
|
|
58
101
|
{
|
|
59
|
-
|
|
60
|
-
|
|
102
|
+
docId: {
|
|
103
|
+
equals: docId
|
|
104
|
+
}
|
|
61
105
|
}
|
|
62
106
|
]
|
|
63
107
|
}
|
|
64
108
|
});
|
|
65
|
-
const idsToDelete = (results.matches || []).map((match)=>match.id);
|
|
66
|
-
if (idsToDelete.length > 0) {
|
|
67
|
-
await vectorizeBinding.deleteByIds(idsToDelete);
|
|
68
|
-
}
|
|
69
109
|
} catch (error) {
|
|
70
|
-
const errorMessage = error.message
|
|
110
|
+
const errorMessage = error instanceof Error ? error.message : String(error);
|
|
71
111
|
payload.logger.error(`[@payloadcms-vectorize/cf] Failed to delete embeddings: ${errorMessage}`);
|
|
72
112
|
throw new Error(`[@payloadcms-vectorize/cf] Failed to delete embeddings: ${errorMessage}`);
|
|
73
113
|
}
|
|
@@ -77,5 +117,6 @@ import search from './search.js';
|
|
|
77
117
|
adapter
|
|
78
118
|
};
|
|
79
119
|
};
|
|
120
|
+
export { CF_MAPPINGS_SLUG } from './collections/cfMappings.js';
|
|
80
121
|
|
|
81
122
|
//# sourceMappingURL=index.js.map
|
package/dist/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/index.ts"],"sourcesContent":["import type { DbAdapter } from 'payloadcms-vectorize'\nimport type { CloudflareVectorizeBinding, KnowledgePoolsConfig } from './types.js'\nimport embed from './embed.js'\nimport search from './search.js'\n\n/**\n * Configuration for Cloudflare Vectorize integration\n */\ninterface CloudflareVectorizeConfig {\n /** Knowledge pools configuration with their dimensions */\n config: KnowledgePoolsConfig\n /** Cloudflare Vectorize binding for vector storage */\n binding: CloudflareVectorizeBinding\n}\n\n/**\n * Create a Cloudflare Vectorize integration for payloadcms-vectorize\n *\n * @param options Configuration object with knowledge pools and Vectorize binding\n * @returns Object containing the DbAdapter instance\n *\n * @example\n * ```typescript\n * import { createCloudflareVectorizeIntegration } from '@payloadcms-vectorize/cf'\n *\n * const { adapter } = createCloudflareVectorizeIntegration({\n * config: {\n * default: {\n * dims: 384,\n * },\n * },\n * binding: env.VECTORIZE,\n * })\n * ```\n */\nexport const createCloudflareVectorizeIntegration = (\n options: CloudflareVectorizeConfig,\n): { adapter: DbAdapter } => {\n if (!options.binding) {\n throw new Error('[@payloadcms-vectorize/cf] Cloudflare Vectorize binding is required')\n }\n\n const poolConfig = options.config\n\n const adapter: DbAdapter = {\n getConfigExtension: () => {\n return {\n custom: {\n _cfVectorizeAdapter: true,\n _poolConfigs: poolConfig,\n _vectorizeBinding: options.binding,\n },\n }\n },\n\n search
|
|
1
|
+
{"version":3,"sources":["../src/index.ts"],"sourcesContent":["import type { CollectionSlug } from 'payload'\nimport type { DbAdapter } from 'payloadcms-vectorize'\nimport { getVectorizeBinding } from './types.js'\nimport type { CloudflareVectorizeBinding, KnowledgePoolsConfig } from './types.js'\nimport cfMappingsCollection, { CF_MAPPINGS_SLUG } from './collections/cfMappings.js'\nimport embed from './embed.js'\nimport search from './search.js'\n\n/**\n * Configuration for Cloudflare Vectorize integration\n */\ninterface CloudflareVectorizeConfig {\n /** Knowledge pools configuration with their dimensions */\n config: KnowledgePoolsConfig\n /** Cloudflare Vectorize binding for vector storage */\n binding: CloudflareVectorizeBinding\n}\n\n/**\n * Create a Cloudflare Vectorize integration for payloadcms-vectorize\n *\n * @param options Configuration object with knowledge pools and Vectorize binding\n * @returns Object containing the DbAdapter instance\n *\n * @example\n * ```typescript\n * import { createCloudflareVectorizeIntegration } from '@payloadcms-vectorize/cf'\n *\n * const { adapter } = createCloudflareVectorizeIntegration({\n * config: {\n * default: {\n * dims: 384,\n * },\n * },\n * binding: env.VECTORIZE,\n * })\n * ```\n */\nexport const createCloudflareVectorizeIntegration = (\n options: CloudflareVectorizeConfig,\n): { adapter: DbAdapter } => {\n if (!options.binding) {\n throw new Error('[@payloadcms-vectorize/cf] Cloudflare Vectorize binding is required')\n }\n\n const poolConfig = options.config\n\n const adapter: DbAdapter = {\n getConfigExtension: () => {\n return {\n collections: {\n [CF_MAPPINGS_SLUG]: cfMappingsCollection,\n },\n custom: {\n _cfVectorizeAdapter: true,\n _poolConfigs: poolConfig,\n _vectorizeBinding: options.binding,\n },\n }\n },\n\n search,\n\n storeEmbedding: embed,\n\n deleteEmbeddings: async (payload, poolName, sourceCollection, docId) => {\n const vectorizeBinding = getVectorizeBinding(payload)\n\n try {\n // Paginate through all mapping rows for this document+pool\n const allVectorIds: string[] = []\n let page = 1\n let hasNextPage = true\n\n while (hasNextPage) {\n const mappings = await payload.find({\n collection: CF_MAPPINGS_SLUG as CollectionSlug,\n where: {\n and: [\n { poolName: { equals: poolName } },\n { sourceCollection: { equals: sourceCollection } },\n { docId: { equals: docId } },\n ],\n },\n page,\n })\n\n for (const mapping of mappings.docs) {\n allVectorIds.push((mapping as Record<string, unknown>).vectorId as string)\n }\n\n hasNextPage = mappings.hasNextPage\n page++\n }\n\n if (allVectorIds.length === 0) {\n return\n }\n // Delete vectors from Cloudflare Vectorize\n await vectorizeBinding.deleteByIds(allVectorIds)\n // Delete mapping rows\n await payload.delete({\n collection: CF_MAPPINGS_SLUG as CollectionSlug,\n where: {\n and: [\n { poolName: { equals: poolName } },\n { sourceCollection: { equals: sourceCollection } },\n { docId: { equals: docId } },\n ],\n },\n })\n } catch (error) {\n const errorMessage = error instanceof Error ? error.message : String(error)\n payload.logger.error(\n `[@payloadcms-vectorize/cf] Failed to delete embeddings: ${errorMessage}`,\n )\n throw new Error(`[@payloadcms-vectorize/cf] Failed to delete embeddings: ${errorMessage}`)\n }\n },\n }\n\n return { adapter }\n}\n\nexport { CF_MAPPINGS_SLUG } from './collections/cfMappings.js'\nexport type { CloudflareVectorizeBinding, KnowledgePoolsConfig }\nexport type { KnowledgePoolsConfig as KnowledgePoolConfig }\n"],"names":["getVectorizeBinding","cfMappingsCollection","CF_MAPPINGS_SLUG","embed","search","createCloudflareVectorizeIntegration","options","binding","Error","poolConfig","config","adapter","getConfigExtension","collections","custom","_cfVectorizeAdapter","_poolConfigs","_vectorizeBinding","storeEmbedding","deleteEmbeddings","payload","poolName","sourceCollection","docId","vectorizeBinding","allVectorIds","page","hasNextPage","mappings","find","collection","where","and","equals","mapping","docs","push","vectorId","length","deleteByIds","delete","error","errorMessage","message","String","logger"],"mappings":"AAEA,SAASA,mBAAmB,QAAQ,aAAY;AAEhD,OAAOC,wBAAwBC,gBAAgB,QAAQ,8BAA6B;AACpF,OAAOC,WAAW,aAAY;AAC9B,OAAOC,YAAY,cAAa;AAYhC;;;;;;;;;;;;;;;;;;;CAmBC,GACD,OAAO,MAAMC,uCAAuC,CAClDC;IAEA,IAAI,CAACA,QAAQC,OAAO,EAAE;QACpB,MAAM,IAAIC,MAAM;IAClB;IAEA,MAAMC,aAAaH,QAAQI,MAAM;IAEjC,MAAMC,UAAqB;QACzBC,oBAAoB;YAClB,OAAO;gBACLC,aAAa;oBACX,CAACX,iBAAiB,EAAED;gBACtB;gBACAa,QAAQ;oBACNC,qBAAqB;oBACrBC,cAAcP;oBACdQ,mBAAmBX,QAAQC,OAAO;gBACpC;YACF;QACF;QAEAH;QAEAc,gBAAgBf;QAEhBgB,kBAAkB,OAAOC,SAASC,UAAUC,kBAAkBC;YAC5D,MAAMC,mBAAmBxB,oBAAoBoB;YAE7C,IAAI;gBACF,2DAA2D;gBAC3D,MAAMK,eAAyB,EAAE;gBACjC,IAAIC,OAAO;gBACX,IAAIC,cAAc;gBAElB,MAAOA,YAAa;oBAClB,MAAMC,WAAW,MAAMR,QAAQS,IAAI,CAAC;wBAClCC,YAAY5B;wBACZ6B,OAAO;4BACLC,KAAK;gCACH;oCAAEX,UAAU;wCAAEY,QAAQZ;oCAAS;gCAAE;gCACjC;oCAAEC,kBAAkB;wCAAEW,QAAQX;oCAAiB;gCAAE;gCACjD;oCAAEC,OAAO;wCAAEU,QAAQV;oCAAM;gCAAE;6BAC5B;wBACH;wBACAG;oBACF;oBAEA,KAAK,MAAMQ,WAAWN,SAASO,IAAI,CAAE;wBACnCV,aAAaW,IAAI,CAAC,AAACF,QAAoCG,QAAQ;oBACjE;oBAEAV,cAAcC,SAASD,WAAW;oBAClCD;gBACF;gBAEA,IAAID,aAAaa,MAAM,KAAK,GAAG;oBAC7B;gBACF;gBACA,2CAA2C;gBAC3C,MAAMd,iBAAiBe,WAAW,CAACd;gBACnC,sBAAsB;gBACtB,MAAML,QAAQoB,MAAM,CAAC;oBACnBV,YAAY5B;oBACZ6B,OAAO;wBACLC,KAAK;4BACH;gCAAEX,UAAU;oCAAEY,QAAQZ;gCAAS;4BAAE;4BACjC;gCAAEC,kBAAkB;oCAAEW,QAAQX;gCAAiB;4BAAE;4BACjD;gCAAEC,OAAO;oCAAEU,QAAQV;gCAAM;4BAAE;yBAC5B;oBACH;gBACF;YACF,EAAE,OAAOkB,OAAO;gBACd,MAAMC,eAAeD,iBAAiBjC,QAAQiC,MAAME,OAAO,GAAGC,OAAOH;gBACrErB,QAAQyB,MAAM,CAACJ,KAAK,CAClB,CAAC,wDAAwD,EAAEC,cAAc;gBAE3E,MAAM,IAAIlC,MAAM,CAAC,wDAAwD,EAAEkC,cAAc;YAC3F;QACF;IACF;IAEA,OAAO;QAAE/B;IAAQ;AACnB,EAAC;AAED,SAAST,gBAAgB,QAAQ,8BAA6B"}
|
package/dist/search.js
CHANGED
|
@@ -1,18 +1,9 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { getVectorizeBinding } from './types.js';
|
|
2
2
|
/**
|
|
3
3
|
* Search for similar vectors in Cloudflare Vectorize
|
|
4
4
|
*/ export default (async (payload, queryEmbedding, poolName, limit = 10, where)=>{
|
|
5
|
-
|
|
6
|
-
const vectorizeBinding = getVectorizedPayload(payload)?.getDbAdapterCustom()?._vectorizeBinding;
|
|
7
|
-
if (!vectorizeBinding) {
|
|
8
|
-
throw new Error('[@payloadcms-vectorize/cf] Cloudflare Vectorize binding not found');
|
|
9
|
-
}
|
|
5
|
+
const vectorizeBinding = getVectorizeBinding(payload);
|
|
10
6
|
try {
|
|
11
|
-
// Get collection config
|
|
12
|
-
const collectionConfig = payload.collections[poolName]?.config;
|
|
13
|
-
if (!collectionConfig) {
|
|
14
|
-
throw new Error(`Collection ${poolName} not found`);
|
|
15
|
-
}
|
|
16
7
|
// Query Cloudflare Vectorize
|
|
17
8
|
// The query returns the top-k most similar vectors
|
|
18
9
|
const results = await vectorizeBinding.query(queryEmbedding, {
|
|
@@ -22,36 +13,52 @@ import { getVectorizedPayload } from 'payloadcms-vectorize';
|
|
|
22
13
|
if (!results.matches) {
|
|
23
14
|
return [];
|
|
24
15
|
}
|
|
25
|
-
//
|
|
26
|
-
const
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
}
|
|
41
|
-
}
|
|
42
|
-
|
|
43
|
-
|
|
16
|
+
// Batch-fetch all matched documents, paginating through results
|
|
17
|
+
const matchIds = results.matches.map((m)=>m.id);
|
|
18
|
+
const scoreById = new Map(results.matches.map((m)=>[
|
|
19
|
+
m.id,
|
|
20
|
+
m.score || 0
|
|
21
|
+
]));
|
|
22
|
+
const docsById = new Map();
|
|
23
|
+
let page = 1;
|
|
24
|
+
let hasNextPage = true;
|
|
25
|
+
while(hasNextPage){
|
|
26
|
+
const found = await payload.find({
|
|
27
|
+
collection: poolName,
|
|
28
|
+
where: {
|
|
29
|
+
id: {
|
|
30
|
+
in: matchIds
|
|
31
|
+
}
|
|
32
|
+
},
|
|
33
|
+
page
|
|
34
|
+
});
|
|
35
|
+
for (const doc of found.docs){
|
|
36
|
+
docsById.set(String(doc.id), doc);
|
|
44
37
|
}
|
|
38
|
+
hasNextPage = found.hasNextPage;
|
|
39
|
+
page++;
|
|
40
|
+
}
|
|
41
|
+
// Build results preserving the original similarity-score order
|
|
42
|
+
const searchResults = [];
|
|
43
|
+
for (const matchId of matchIds){
|
|
44
|
+
const doc = docsById.get(matchId);
|
|
45
|
+
if (!doc || where && !matchesWhere(doc, where)) continue;
|
|
46
|
+
const { id: _id, createdAt: _createdAt, updatedAt: _updatedAt, ...docFields } = doc;
|
|
47
|
+
searchResults.push({
|
|
48
|
+
id: matchId,
|
|
49
|
+
score: scoreById.get(matchId) || 0,
|
|
50
|
+
...docFields
|
|
51
|
+
});
|
|
45
52
|
}
|
|
46
53
|
return searchResults;
|
|
47
54
|
} catch (e) {
|
|
48
|
-
const errorMessage = e.message
|
|
55
|
+
const errorMessage = e instanceof Error ? e.message : String(e);
|
|
49
56
|
payload.logger.error(`[@payloadcms-vectorize/cf] Search failed: ${errorMessage}`);
|
|
50
57
|
throw new Error(`[@payloadcms-vectorize/cf] Search failed: ${errorMessage}`);
|
|
51
58
|
}
|
|
52
59
|
});
|
|
53
60
|
/**
|
|
54
|
-
* Simple WHERE clause matcher for basic filtering
|
|
61
|
+
* Simple WHERE clause matcher for basic filtering.
|
|
55
62
|
* Supports: equals, in, exists, and, or
|
|
56
63
|
*/ function matchesWhere(doc, where) {
|
|
57
64
|
if (!where || Object.keys(where).length === 0) return true;
|
package/dist/search.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/search.ts"],"sourcesContent":["import { BasePayload, Where } from 'payload'\nimport { KnowledgePoolName, VectorSearchResult
|
|
1
|
+
{"version":3,"sources":["../src/search.ts"],"sourcesContent":["import { BasePayload, CollectionSlug, Where } from 'payload'\nimport { KnowledgePoolName, VectorSearchResult } from 'payloadcms-vectorize'\nimport { getVectorizeBinding } from './types.js'\n\n/**\n * Search for similar vectors in Cloudflare Vectorize\n */\nexport default async (\n payload: BasePayload,\n queryEmbedding: number[],\n poolName: KnowledgePoolName,\n limit: number = 10,\n where?: Where,\n): Promise<Array<VectorSearchResult>> => {\n const vectorizeBinding = getVectorizeBinding(payload)\n\n try {\n // Query Cloudflare Vectorize\n // The query returns the top-k most similar vectors\n const results = await vectorizeBinding.query(queryEmbedding, {\n topK: limit,\n returnMetadata: true,\n })\n\n if (!results.matches) {\n return []\n }\n\n // Batch-fetch all matched documents, paginating through results\n const matchIds = results.matches.map((m) => m.id)\n const scoreById = new Map(results.matches.map((m) => [m.id, m.score || 0]))\n\n const docsById = new Map<string, Record<string, unknown>>()\n let page = 1\n let hasNextPage = true\n while (hasNextPage) {\n const found = await payload.find({\n collection: poolName as CollectionSlug,\n where: { id: { in: matchIds } },\n page,\n })\n for (const doc of found.docs as Record<string, unknown>[]) {\n docsById.set(String(doc.id), doc)\n }\n hasNextPage = found.hasNextPage\n page++\n }\n\n // Build results preserving the original similarity-score order\n const searchResults: VectorSearchResult[] = []\n for (const matchId of matchIds) {\n const doc = docsById.get(matchId)\n if (!doc || (where && !matchesWhere(doc, where))) continue\n\n const { id: _id, createdAt: _createdAt, updatedAt: _updatedAt, ...docFields } = doc\n searchResults.push({\n id: matchId,\n score: scoreById.get(matchId) || 0,\n ...docFields,\n } as VectorSearchResult)\n }\n\n return searchResults\n } catch (e) {\n const errorMessage = e instanceof Error ? e.message : String(e)\n payload.logger.error(`[@payloadcms-vectorize/cf] Search failed: ${errorMessage}`)\n throw new Error(`[@payloadcms-vectorize/cf] Search failed: ${errorMessage}`)\n }\n}\n\n/**\n * Simple WHERE clause matcher for basic filtering.\n * Supports: equals, in, exists, and, or\n */\nfunction matchesWhere(doc: Record<string, unknown>, where: Where): boolean {\n if (!where || Object.keys(where).length === 0) return true\n\n // Handle 'and' operator\n if ('and' in where && Array.isArray(where.and)) {\n return where.and.every((clause: Where) => matchesWhere(doc, clause))\n }\n\n // Handle 'or' operator\n if ('or' in where && Array.isArray(where.or)) {\n return where.or.some((clause: Where) => matchesWhere(doc, clause))\n }\n\n // Handle field-level conditions\n for (const [field, condition] of Object.entries(where)) {\n if (field === 'and' || field === 'or') continue\n\n const value = doc[field]\n\n if (typeof condition === 'object' && condition !== null) {\n if ('equals' in condition && value !== condition.equals) {\n return false\n }\n if ('in' in condition && Array.isArray(condition.in) && !condition.in.includes(value)) {\n return false\n }\n if ('exists' in condition) {\n const exists = value !== undefined && value !== null\n if (condition.exists !== exists) return false\n }\n }\n }\n\n return true\n}\n"],"names":["getVectorizeBinding","payload","queryEmbedding","poolName","limit","where","vectorizeBinding","results","query","topK","returnMetadata","matches","matchIds","map","m","id","scoreById","Map","score","docsById","page","hasNextPage","found","find","collection","in","doc","docs","set","String","searchResults","matchId","get","matchesWhere","_id","createdAt","_createdAt","updatedAt","_updatedAt","docFields","push","e","errorMessage","Error","message","logger","error","Object","keys","length","Array","isArray","and","every","clause","or","some","field","condition","entries","value","equals","includes","exists","undefined"],"mappings":"AAEA,SAASA,mBAAmB,QAAQ,aAAY;AAEhD;;CAEC,GACD,eAAe,CAAA,OACbC,SACAC,gBACAC,UACAC,QAAgB,EAAE,EAClBC;IAEA,MAAMC,mBAAmBN,oBAAoBC;IAE7C,IAAI;QACF,6BAA6B;QAC7B,mDAAmD;QACnD,MAAMM,UAAU,MAAMD,iBAAiBE,KAAK,CAACN,gBAAgB;YAC3DO,MAAML;YACNM,gBAAgB;QAClB;QAEA,IAAI,CAACH,QAAQI,OAAO,EAAE;YACpB,OAAO,EAAE;QACX;QAEA,gEAAgE;QAChE,MAAMC,WAAWL,QAAQI,OAAO,CAACE,GAAG,CAAC,CAACC,IAAMA,EAAEC,EAAE;QAChD,MAAMC,YAAY,IAAIC,IAAIV,QAAQI,OAAO,CAACE,GAAG,CAAC,CAACC,IAAM;gBAACA,EAAEC,EAAE;gBAAED,EAAEI,KAAK,IAAI;aAAE;QAEzE,MAAMC,WAAW,IAAIF;QACrB,IAAIG,OAAO;QACX,IAAIC,cAAc;QAClB,MAAOA,YAAa;YAClB,MAAMC,QAAQ,MAAMrB,QAAQsB,IAAI,CAAC;gBAC/BC,YAAYrB;gBACZE,OAAO;oBAAEU,IAAI;wBAAEU,IAAIb;oBAAS;gBAAE;gBAC9BQ;YACF;YACA,KAAK,MAAMM,OAAOJ,MAAMK,IAAI,CAA+B;gBACzDR,SAASS,GAAG,CAACC,OAAOH,IAAIX,EAAE,GAAGW;YAC/B;YACAL,cAAcC,MAAMD,WAAW;YAC/BD;QACF;QAEA,+DAA+D;QAC/D,MAAMU,gBAAsC,EAAE;QAC9C,KAAK,MAAMC,WAAWnB,SAAU;YAC9B,MAAMc,MAAMP,SAASa,GAAG,CAACD;YACzB,IAAI,CAACL,OAAQrB,SAAS,CAAC4B,aAAaP,KAAKrB,QAAS;YAElD,MAAM,EAAEU,IAAImB,GAAG,EAAEC,WAAWC,UAAU,EAAEC,WAAWC,UAAU,EAAE,GAAGC,WAAW,GAAGb;YAChFI,cAAcU,IAAI,CAAC;gBACjBzB,IAAIgB;gBACJb,OAAOF,UAAUgB,GAAG,CAACD,YAAY;gBACjC,GAAGQ,SAAS;YACd;QACF;QAEA,OAAOT;IACT,EAAE,OAAOW,GAAG;QACV,MAAMC,eAAeD,aAAaE,QAAQF,EAAEG,OAAO,GAAGf,OAAOY;QAC7DxC,QAAQ4C,MAAM,CAACC,KAAK,CAAC,CAAC,0CAA0C,EAAEJ,cAAc;QAChF,MAAM,IAAIC,MAAM,CAAC,0CAA0C,EAAED,cAAc;IAC7E;AACF,CAAA,EAAC;AAED;;;CAGC,GACD,SAAST,aAAaP,GAA4B,EAAErB,KAAY;IAC9D,IAAI,CAACA,SAAS0C,OAAOC,IAAI,CAAC3C,OAAO4C,MAAM,KAAK,GAAG,OAAO;IAEtD,wBAAwB;IACxB,IAAI,SAAS5C,SAAS6C,MAAMC,OAAO,CAAC9C,MAAM+C,GAAG,GAAG;QAC9C,OAAO/C,MAAM+C,GAAG,CAACC,KAAK,CAAC,CAACC,SAAkBrB,aAAaP,KAAK4B;IAC9D;IAEA,uBAAuB;IACvB,IAAI,QAAQjD,SAAS6C,MAAMC,OAAO,CAAC9C,MAAMkD,EAAE,GAAG;QAC5C,OAAOlD,MAAMkD,EAAE,CAACC,IAAI,CAAC,CAACF,SAAkBrB,aAAaP,KAAK4B;IAC5D;IAEA,gCAAgC;IAChC,KAAK,MAAM,CAACG,OAAOC,UAAU,IAAIX,OAAOY,OAAO,CAACtD,OAAQ;QACtD,IAAIoD,UAAU,SAASA,UAAU,MAAM;QAEvC,MAAMG,QAAQlC,GAAG,CAAC+B,MAAM;QAExB,IAAI,OAAOC,cAAc,YAAYA,cAAc,MAAM;YACvD,IAAI,YAAYA,aAAaE,UAAUF,UAAUG,MAAM,EAAE;gBACvD,OAAO;YACT;YACA,IAAI,QAAQH,aAAaR,MAAMC,OAAO,CAACO,UAAUjC,EAAE,KAAK,CAACiC,UAAUjC,EAAE,CAACqC,QAAQ,CAACF,QAAQ;gBACrF,OAAO;YACT;YACA,IAAI,YAAYF,WAAW;gBACzB,MAAMK,SAASH,UAAUI,aAAaJ,UAAU;gBAChD,IAAIF,UAAUK,MAAM,KAAKA,QAAQ,OAAO;YAC1C;QACF;IACF;IAEA,OAAO;AACT"}
|
package/dist/types.js
CHANGED
|
@@ -1,5 +1,13 @@
|
|
|
1
|
+
import { getVectorizedPayload } from 'payloadcms-vectorize';
|
|
1
2
|
/**
|
|
2
|
-
* Cloudflare Vectorize binding
|
|
3
|
-
|
|
3
|
+
* Retrieve the Cloudflare Vectorize binding from a Payload instance.
|
|
4
|
+
* Throws if the binding is not found.
|
|
5
|
+
*/ export function getVectorizeBinding(payload) {
|
|
6
|
+
const binding = getVectorizedPayload(payload)?.getDbAdapterCustom()?._vectorizeBinding;
|
|
7
|
+
if (!binding) {
|
|
8
|
+
throw new Error('[@payloadcms-vectorize/cf] Cloudflare Vectorize binding not found');
|
|
9
|
+
}
|
|
10
|
+
return binding;
|
|
11
|
+
}
|
|
4
12
|
|
|
5
13
|
//# sourceMappingURL=types.js.map
|
package/dist/types.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"sources":["../src/types.ts"],"sourcesContent":["import { BasePayload } from 'payload'\n\n/**\n * Configuration for a knowledge pool in Cloudflare Vectorize\n */\nexport interface CloudflareVectorizePoolConfig {\n /** Vector dimensions for this pool (must match embedding model output) */\n dims: number\n}\n\n/**\n * All knowledge pools configuration for Cloudflare Vectorize\n */\nexport type KnowledgePoolsConfig = Record<string, CloudflareVectorizePoolConfig>\n\n/**\n * Cloudflare Vectorize binding
|
|
1
|
+
{"version":3,"sources":["../src/types.ts"],"sourcesContent":["import type { BasePayload } from 'payload'\nimport { getVectorizedPayload } from 'payloadcms-vectorize'\n\n/**\n * Retrieve the Cloudflare Vectorize binding from a Payload instance.\n * Throws if the binding is not found.\n */\nexport function getVectorizeBinding(payload: BasePayload): CloudflareVectorizeBinding {\n const binding = getVectorizedPayload(payload)?.getDbAdapterCustom()\n ?._vectorizeBinding as CloudflareVectorizeBinding | undefined\n if (!binding) {\n throw new Error('[@payloadcms-vectorize/cf] Cloudflare Vectorize binding not found')\n }\n return binding\n}\n\n/**\n * Configuration for a knowledge pool in Cloudflare Vectorize\n */\nexport interface CloudflareVectorizePoolConfig {\n /** Vector dimensions for this pool (must match embedding model output) */\n dims: number\n}\n\n/**\n * All knowledge pools configuration for Cloudflare Vectorize\n */\nexport type KnowledgePoolsConfig = Record<string, CloudflareVectorizePoolConfig>\n\n/** A single vector match returned by a Vectorize query */\nexport interface VectorizeMatch {\n id: string\n score?: number\n metadata?: Record<string, unknown>\n}\n\n/** Result of a Vectorize query */\nexport interface VectorizeQueryResult {\n matches: VectorizeMatch[]\n count: number\n}\n\n/** Vector to upsert into Vectorize */\nexport interface VectorizeVector {\n id: string\n values: number[]\n metadata?: Record<string, unknown>\n}\n\n/**\n * Cloudflare Vectorize binding interface.\n * Mirrors the subset of the Vectorize API we use.\n * For the full type, install `@cloudflare/workers-types`.\n */\nexport interface CloudflareVectorizeBinding {\n query(vector: number[], options?: {\n topK?: number\n returnMetadata?: boolean | 'indexed' | 'all'\n filter?: Record<string, unknown>\n /** Vectorize metadata filtering */\n where?: Record<string, unknown>\n }): Promise<VectorizeQueryResult>\n upsert(vectors: VectorizeVector[]): Promise<unknown>\n deleteByIds(ids: string[]): Promise<unknown>\n}\n"],"names":["getVectorizedPayload","getVectorizeBinding","payload","binding","getDbAdapterCustom","_vectorizeBinding","Error"],"mappings":"AACA,SAASA,oBAAoB,QAAQ,uBAAsB;AAE3D;;;CAGC,GACD,OAAO,SAASC,oBAAoBC,OAAoB;IACtD,MAAMC,UAAUH,qBAAqBE,UAAUE,sBAC3CC;IACJ,IAAI,CAACF,SAAS;QACZ,MAAM,IAAIG,MAAM;IAClB;IACA,OAAOH;AACT"}
|