graphile-search 2.19.11 → 2.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/adapters/pgvector.js +41 -7
- package/codecs/vector-codec.js +16 -0
- package/esm/adapters/pgvector.js +41 -7
- package/esm/codecs/vector-codec.js +16 -0
- package/esm/plugin.d.ts +7 -0
- package/esm/plugin.js +36 -2
- package/package.json +2 -2
- package/plugin.d.ts +7 -0
- package/plugin.js +37 -2
package/adapters/pgvector.js
CHANGED
|
@@ -49,9 +49,23 @@ function createPgvectorAdapter(options = {}) {
|
|
|
49
49
|
const chunksInfo = enableChunkQuerying ? (0, chunks_1.getChunksInfo)(codec) : undefined;
|
|
50
50
|
for (const [attributeName, attribute] of Object.entries(codec.attributes)) {
|
|
51
51
|
if (isVectorCodec(attribute.codec)) {
|
|
52
|
+
const dimensions = attribute.extensions?.vectorDimensions;
|
|
53
|
+
// `@embeddingModel` on the vector column: the model its definition
|
|
54
|
+
// names. `@embeddingModelOf <column>` on a sibling: the column that
|
|
55
|
+
// records, per row, the model that produced the vector.
|
|
56
|
+
const named = attribute.extensions?.tags?.embeddingModel;
|
|
57
|
+
const model = typeof named === 'string' && named ? named : undefined;
|
|
58
|
+
const modelColumn = Object.entries(codec.attributes).find(([, other]) => other.extensions?.tags?.embeddingModelOf === attributeName)?.[0];
|
|
52
59
|
columns.push({
|
|
53
60
|
attributeName,
|
|
54
|
-
adapterData: chunksInfo
|
|
61
|
+
adapterData: chunksInfo || dimensions || model || modelColumn
|
|
62
|
+
? {
|
|
63
|
+
...(chunksInfo && { chunksInfo }),
|
|
64
|
+
...(dimensions && { dimensions }),
|
|
65
|
+
...(model && { model }),
|
|
66
|
+
...(modelColumn && { modelColumn }),
|
|
67
|
+
}
|
|
68
|
+
: undefined,
|
|
55
69
|
});
|
|
56
70
|
}
|
|
57
71
|
}
|
|
@@ -96,6 +110,12 @@ function createPgvectorAdapter(options = {}) {
|
|
|
96
110
|
type: new GraphQLNonNull(new GraphQLList(new GraphQLNonNull(GraphQLFloat))),
|
|
97
111
|
description: 'Query vector for similarity search.',
|
|
98
112
|
},
|
|
113
|
+
model: {
|
|
114
|
+
type: build.graphql.GraphQLString,
|
|
115
|
+
description: 'The model that produced `vector`. On a column that records the model of each ' +
|
|
116
|
+
'row\'s vector, only rows embedded with this model are compared; vectors of ' +
|
|
117
|
+
'two models are not comparable. Filled in server-side when `text` is embedded.',
|
|
118
|
+
},
|
|
99
119
|
metric: {
|
|
100
120
|
type: VectorMetricEnum,
|
|
101
121
|
description: `Similarity metric to use (default: ${defaultMetric}).`,
|
|
@@ -124,7 +144,7 @@ function createPgvectorAdapter(options = {}) {
|
|
|
124
144
|
buildFilterApply(sql, alias, column, filterValue, _build) {
|
|
125
145
|
if (filterValue == null)
|
|
126
146
|
return null;
|
|
127
|
-
const { vector, metric, distance, includeChunks } = filterValue;
|
|
147
|
+
const { vector, metric, distance, includeChunks, model } = filterValue;
|
|
128
148
|
if (!vector || !Array.isArray(vector) || vector.length === 0)
|
|
129
149
|
return null;
|
|
130
150
|
const resolvedMetric = metric || defaultMetric;
|
|
@@ -133,6 +153,16 @@ function createPgvectorAdapter(options = {}) {
|
|
|
133
153
|
// Check if this column has chunks info and chunk querying is requested
|
|
134
154
|
const adapterData = column.adapterData;
|
|
135
155
|
const chunksInfo = adapterData?.chunksInfo;
|
|
156
|
+
// A row whose vector another model produced is not a worse match, it is
|
|
157
|
+
// not a match at all: its distance to this query vector means nothing.
|
|
158
|
+
const sameModel = adapterData?.modelColumn && typeof model === 'string' && model
|
|
159
|
+
? sql `${alias}.${sql.identifier(adapterData.modelColumn)} = ${sql.value(model)}`
|
|
160
|
+
: null;
|
|
161
|
+
// Narrows a clause that is there; where there is none, `scorableClause`
|
|
162
|
+
// carries the model so a search that only ranks (unifiedSearch) is not
|
|
163
|
+
// turned into one that filters.
|
|
164
|
+
const both = (a, b) => (a && b ? sql `(${a}) AND (${b})` : b);
|
|
165
|
+
const scored = (distance) => sameModel ? sql `CASE WHEN ${sameModel} THEN ${distance} END` : distance;
|
|
136
166
|
if (chunksInfo && (includeChunks !== false)) {
|
|
137
167
|
// Chunk-aware query: find the closest chunk for each parent row
|
|
138
168
|
// Uses a lateral subquery to get the minimum distance across all chunks
|
|
@@ -154,13 +184,17 @@ function createPgvectorAdapter(options = {}) {
|
|
|
154
184
|
)`;
|
|
155
185
|
// Direct parent distance (NULL when the parent has no embedding)
|
|
156
186
|
const parentColumnExpr = sql `${alias}.${sql.identifier(column.attributeName)}`;
|
|
157
|
-
|
|
187
|
+
// The model stamp is the parent vector's; a chunk's vector is not the parent's.
|
|
188
|
+
const parentDistanceExpr = scored(buildDistanceExpr(sql, parentColumnExpr, vectorExpr, resolvedMetric));
|
|
189
|
+
const parentScorable = sameModel
|
|
190
|
+
? sql `(${sameModel} AND ${parentColumnExpr} IS NOT NULL)`
|
|
191
|
+
: sql `${parentColumnExpr} IS NOT NULL`;
|
|
158
192
|
// LEAST skips NULL operands, so this is the closest of parent + chunks
|
|
159
193
|
// and NULL only when neither the parent nor any chunk is embedded.
|
|
160
194
|
const combinedDistanceExpr = sql `LEAST(${parentDistanceExpr}, ${chunkDistanceSubquery})`;
|
|
161
195
|
// Rows with no embedding anywhere have no distance to rank: leave them out.
|
|
162
196
|
// MIN(distance) is NULL exactly when no chunk has an embedding.
|
|
163
|
-
const scorableClause = sql `(${
|
|
197
|
+
const scorableClause = sql `(${parentScorable} OR ${chunkDistanceSubquery} IS NOT NULL)`;
|
|
164
198
|
let whereClause = null;
|
|
165
199
|
if (distance !== undefined && distance !== null) {
|
|
166
200
|
whereClause = sql `${combinedDistanceExpr} <= ${sql.value(distance)}`;
|
|
@@ -179,10 +213,10 @@ function createPgvectorAdapter(options = {}) {
|
|
|
179
213
|
whereClause = sql `${distanceExpr} <= ${sql.value(distance)}`;
|
|
180
214
|
}
|
|
181
215
|
return {
|
|
182
|
-
whereClause,
|
|
183
|
-
scoreExpression: distanceExpr,
|
|
216
|
+
whereClause: both(sameModel, whereClause),
|
|
217
|
+
scoreExpression: scored(distanceExpr),
|
|
184
218
|
// A NULL embedding has no distance to rank: leave the row out.
|
|
185
|
-
scorableClause: sql `${columnExpr} IS NOT NULL
|
|
219
|
+
scorableClause: both(sameModel, sql `${columnExpr} IS NOT NULL`),
|
|
186
220
|
};
|
|
187
221
|
},
|
|
188
222
|
};
|
package/codecs/vector-codec.js
CHANGED
|
@@ -60,6 +60,22 @@ exports.VectorCodecPlugin = {
|
|
|
60
60
|
},
|
|
61
61
|
};
|
|
62
62
|
},
|
|
63
|
+
/**
|
|
64
|
+
* Record a vector column's declared length (`vector(512)` → 512; pgvector's
|
|
65
|
+
* type modifier is the length itself) on the attribute, so a search that
|
|
66
|
+
* holds one query vector can tell which columns it can be compared with.
|
|
67
|
+
*/
|
|
68
|
+
async pgCodecs_attribute(_info, event) {
|
|
69
|
+
const { pgAttribute, attribute } = event;
|
|
70
|
+
if (attribute.codec?.name !== 'vector')
|
|
71
|
+
return;
|
|
72
|
+
const typmod = pgAttribute.atttypmod;
|
|
73
|
+
if (typmod == null || typmod < 1)
|
|
74
|
+
return;
|
|
75
|
+
if (!attribute.extensions)
|
|
76
|
+
attribute.extensions = {};
|
|
77
|
+
attribute.extensions.vectorDimensions = typmod;
|
|
78
|
+
},
|
|
63
79
|
},
|
|
64
80
|
},
|
|
65
81
|
schema: {
|
package/esm/adapters/pgvector.js
CHANGED
|
@@ -46,9 +46,23 @@ export function createPgvectorAdapter(options = {}) {
|
|
|
46
46
|
const chunksInfo = enableChunkQuerying ? getChunksInfo(codec) : undefined;
|
|
47
47
|
for (const [attributeName, attribute] of Object.entries(codec.attributes)) {
|
|
48
48
|
if (isVectorCodec(attribute.codec)) {
|
|
49
|
+
const dimensions = attribute.extensions?.vectorDimensions;
|
|
50
|
+
// `@embeddingModel` on the vector column: the model its definition
|
|
51
|
+
// names. `@embeddingModelOf <column>` on a sibling: the column that
|
|
52
|
+
// records, per row, the model that produced the vector.
|
|
53
|
+
const named = attribute.extensions?.tags?.embeddingModel;
|
|
54
|
+
const model = typeof named === 'string' && named ? named : undefined;
|
|
55
|
+
const modelColumn = Object.entries(codec.attributes).find(([, other]) => other.extensions?.tags?.embeddingModelOf === attributeName)?.[0];
|
|
49
56
|
columns.push({
|
|
50
57
|
attributeName,
|
|
51
|
-
adapterData: chunksInfo
|
|
58
|
+
adapterData: chunksInfo || dimensions || model || modelColumn
|
|
59
|
+
? {
|
|
60
|
+
...(chunksInfo && { chunksInfo }),
|
|
61
|
+
...(dimensions && { dimensions }),
|
|
62
|
+
...(model && { model }),
|
|
63
|
+
...(modelColumn && { modelColumn }),
|
|
64
|
+
}
|
|
65
|
+
: undefined,
|
|
52
66
|
});
|
|
53
67
|
}
|
|
54
68
|
}
|
|
@@ -93,6 +107,12 @@ export function createPgvectorAdapter(options = {}) {
|
|
|
93
107
|
type: new GraphQLNonNull(new GraphQLList(new GraphQLNonNull(GraphQLFloat))),
|
|
94
108
|
description: 'Query vector for similarity search.',
|
|
95
109
|
},
|
|
110
|
+
model: {
|
|
111
|
+
type: build.graphql.GraphQLString,
|
|
112
|
+
description: 'The model that produced `vector`. On a column that records the model of each ' +
|
|
113
|
+
'row\'s vector, only rows embedded with this model are compared; vectors of ' +
|
|
114
|
+
'two models are not comparable. Filled in server-side when `text` is embedded.',
|
|
115
|
+
},
|
|
96
116
|
metric: {
|
|
97
117
|
type: VectorMetricEnum,
|
|
98
118
|
description: `Similarity metric to use (default: ${defaultMetric}).`,
|
|
@@ -121,7 +141,7 @@ export function createPgvectorAdapter(options = {}) {
|
|
|
121
141
|
buildFilterApply(sql, alias, column, filterValue, _build) {
|
|
122
142
|
if (filterValue == null)
|
|
123
143
|
return null;
|
|
124
|
-
const { vector, metric, distance, includeChunks } = filterValue;
|
|
144
|
+
const { vector, metric, distance, includeChunks, model } = filterValue;
|
|
125
145
|
if (!vector || !Array.isArray(vector) || vector.length === 0)
|
|
126
146
|
return null;
|
|
127
147
|
const resolvedMetric = metric || defaultMetric;
|
|
@@ -130,6 +150,16 @@ export function createPgvectorAdapter(options = {}) {
|
|
|
130
150
|
// Check if this column has chunks info and chunk querying is requested
|
|
131
151
|
const adapterData = column.adapterData;
|
|
132
152
|
const chunksInfo = adapterData?.chunksInfo;
|
|
153
|
+
// A row whose vector another model produced is not a worse match, it is
|
|
154
|
+
// not a match at all: its distance to this query vector means nothing.
|
|
155
|
+
const sameModel = adapterData?.modelColumn && typeof model === 'string' && model
|
|
156
|
+
? sql `${alias}.${sql.identifier(adapterData.modelColumn)} = ${sql.value(model)}`
|
|
157
|
+
: null;
|
|
158
|
+
// Narrows a clause that is there; where there is none, `scorableClause`
|
|
159
|
+
// carries the model so a search that only ranks (unifiedSearch) is not
|
|
160
|
+
// turned into one that filters.
|
|
161
|
+
const both = (a, b) => (a && b ? sql `(${a}) AND (${b})` : b);
|
|
162
|
+
const scored = (distance) => sameModel ? sql `CASE WHEN ${sameModel} THEN ${distance} END` : distance;
|
|
133
163
|
if (chunksInfo && (includeChunks !== false)) {
|
|
134
164
|
// Chunk-aware query: find the closest chunk for each parent row
|
|
135
165
|
// Uses a lateral subquery to get the minimum distance across all chunks
|
|
@@ -151,13 +181,17 @@ export function createPgvectorAdapter(options = {}) {
|
|
|
151
181
|
)`;
|
|
152
182
|
// Direct parent distance (NULL when the parent has no embedding)
|
|
153
183
|
const parentColumnExpr = sql `${alias}.${sql.identifier(column.attributeName)}`;
|
|
154
|
-
|
|
184
|
+
// The model stamp is the parent vector's; a chunk's vector is not the parent's.
|
|
185
|
+
const parentDistanceExpr = scored(buildDistanceExpr(sql, parentColumnExpr, vectorExpr, resolvedMetric));
|
|
186
|
+
const parentScorable = sameModel
|
|
187
|
+
? sql `(${sameModel} AND ${parentColumnExpr} IS NOT NULL)`
|
|
188
|
+
: sql `${parentColumnExpr} IS NOT NULL`;
|
|
155
189
|
// LEAST skips NULL operands, so this is the closest of parent + chunks
|
|
156
190
|
// and NULL only when neither the parent nor any chunk is embedded.
|
|
157
191
|
const combinedDistanceExpr = sql `LEAST(${parentDistanceExpr}, ${chunkDistanceSubquery})`;
|
|
158
192
|
// Rows with no embedding anywhere have no distance to rank: leave them out.
|
|
159
193
|
// MIN(distance) is NULL exactly when no chunk has an embedding.
|
|
160
|
-
const scorableClause = sql `(${
|
|
194
|
+
const scorableClause = sql `(${parentScorable} OR ${chunkDistanceSubquery} IS NOT NULL)`;
|
|
161
195
|
let whereClause = null;
|
|
162
196
|
if (distance !== undefined && distance !== null) {
|
|
163
197
|
whereClause = sql `${combinedDistanceExpr} <= ${sql.value(distance)}`;
|
|
@@ -176,10 +210,10 @@ export function createPgvectorAdapter(options = {}) {
|
|
|
176
210
|
whereClause = sql `${distanceExpr} <= ${sql.value(distance)}`;
|
|
177
211
|
}
|
|
178
212
|
return {
|
|
179
|
-
whereClause,
|
|
180
|
-
scoreExpression: distanceExpr,
|
|
213
|
+
whereClause: both(sameModel, whereClause),
|
|
214
|
+
scoreExpression: scored(distanceExpr),
|
|
181
215
|
// A NULL embedding has no distance to rank: leave the row out.
|
|
182
|
-
scorableClause: sql `${columnExpr} IS NOT NULL
|
|
216
|
+
scorableClause: both(sameModel, sql `${columnExpr} IS NOT NULL`),
|
|
183
217
|
};
|
|
184
218
|
},
|
|
185
219
|
};
|
|
@@ -54,6 +54,22 @@ export const VectorCodecPlugin = {
|
|
|
54
54
|
},
|
|
55
55
|
};
|
|
56
56
|
},
|
|
57
|
+
/**
|
|
58
|
+
* Record a vector column's declared length (`vector(512)` → 512; pgvector's
|
|
59
|
+
* type modifier is the length itself) on the attribute, so a search that
|
|
60
|
+
* holds one query vector can tell which columns it can be compared with.
|
|
61
|
+
*/
|
|
62
|
+
async pgCodecs_attribute(_info, event) {
|
|
63
|
+
const { pgAttribute, attribute } = event;
|
|
64
|
+
if (attribute.codec?.name !== 'vector')
|
|
65
|
+
return;
|
|
66
|
+
const typmod = pgAttribute.atttypmod;
|
|
67
|
+
if (typmod == null || typmod < 1)
|
|
68
|
+
return;
|
|
69
|
+
if (!attribute.extensions)
|
|
70
|
+
attribute.extensions = {};
|
|
71
|
+
attribute.extensions.vectorDimensions = typmod;
|
|
72
|
+
},
|
|
57
73
|
},
|
|
58
74
|
},
|
|
59
75
|
schema: {
|
package/esm/plugin.d.ts
CHANGED
|
@@ -22,6 +22,13 @@ import 'graphile-connection-filter';
|
|
|
22
22
|
import type { PgCodecWithAttributes } from '@dataplan/pg';
|
|
23
23
|
import type { GraphileConfig } from 'graphile-config';
|
|
24
24
|
import type { UnifiedSearchOptions } from './types';
|
|
25
|
+
/**
|
|
26
|
+
* Carried on a filter field that searches vector columns, for whatever embeds
|
|
27
|
+
* the search text (graphile-llm): which columns, and the model each column's
|
|
28
|
+
* own definition names. Vectors of two models are not comparable, so the text
|
|
29
|
+
* is embedded per model.
|
|
30
|
+
*/
|
|
31
|
+
export declare const VECTOR_COLUMNS_EXT = "graphileSearchVectorColumns";
|
|
25
32
|
declare global {
|
|
26
33
|
namespace GraphileBuild {
|
|
27
34
|
interface Inflection {
|
package/esm/plugin.js
CHANGED
|
@@ -21,6 +21,17 @@ import 'graphile-build-pg';
|
|
|
21
21
|
import 'graphile-connection-filter';
|
|
22
22
|
import { TYPES } from '@dataplan/pg';
|
|
23
23
|
import { getQueryBuilder } from 'graphile-plugin-utils';
|
|
24
|
+
/**
|
|
25
|
+
* Carried on a filter field that searches vector columns, for whatever embeds
|
|
26
|
+
* the search text (graphile-llm): which columns, and the model each column's
|
|
27
|
+
* own definition names. Vectors of two models are not comparable, so the text
|
|
28
|
+
* is embedded per model.
|
|
29
|
+
*/
|
|
30
|
+
export const VECTOR_COLUMNS_EXT = 'graphileSearchVectorColumns';
|
|
31
|
+
function vectorColumnRef(column) {
|
|
32
|
+
const model = column.adapterData?.model;
|
|
33
|
+
return { attributeName: column.attributeName, ...(model && { model }) };
|
|
34
|
+
}
|
|
24
35
|
/**
|
|
25
36
|
* Read the @searchConfig smart tag from a codec's extensions.
|
|
26
37
|
* Returns undefined if no searchConfig tag is present.
|
|
@@ -921,6 +932,9 @@ export function createUnifiedSearchPlugin(options) {
|
|
|
921
932
|
}, {
|
|
922
933
|
description: build.wrapDescription(`${adapter.name.toUpperCase()} search on the \`${column.attributeName}\` column.`, 'field'),
|
|
923
934
|
type: build.getTypeByName(adapter.getFilterTypeName(build)),
|
|
935
|
+
...(adapter.name === 'vector' && {
|
|
936
|
+
extensions: { [VECTOR_COLUMNS_EXT]: [vectorColumnRef(column)] },
|
|
937
|
+
}),
|
|
924
938
|
apply: function plan($condition, val) {
|
|
925
939
|
if (val == null)
|
|
926
940
|
return;
|
|
@@ -967,15 +981,24 @@ export function createUnifiedSearchPlugin(options) {
|
|
|
967
981
|
'When the LLM plugin is active, pgvector also participates via auto-embedding. ' +
|
|
968
982
|
'Rows matching ANY algorithm are returned. All matching score fields are populated.', 'field'),
|
|
969
983
|
type: build.graphql.GraphQLString,
|
|
984
|
+
extensions: {
|
|
985
|
+
[VECTOR_COLUMNS_EXT]: vectorAdapterColumns.flatMap((ac) => ac.columns.map(vectorColumnRef)),
|
|
986
|
+
},
|
|
970
987
|
apply: function plan($condition, val) {
|
|
971
988
|
if (val == null)
|
|
972
989
|
return;
|
|
973
990
|
// Support both plain string and { __text, __vector } from LLM plugin
|
|
974
991
|
let text;
|
|
975
992
|
let vector = null;
|
|
993
|
+
let model;
|
|
994
|
+
// Absent: one vector for every column. Present: a column it does
|
|
995
|
+
// not name has no vector of its own model and sits this search out.
|
|
996
|
+
let perColumn;
|
|
976
997
|
if (typeof val === 'object' && val.__text) {
|
|
977
998
|
text = val.__text;
|
|
978
999
|
vector = val.__vector ?? null;
|
|
1000
|
+
model = val.__model ?? undefined;
|
|
1001
|
+
perColumn = val.__vectors ?? undefined;
|
|
979
1002
|
}
|
|
980
1003
|
else {
|
|
981
1004
|
text = typeof val === 'string' ? val : String(val);
|
|
@@ -1001,10 +1024,21 @@ export function createUnifiedSearchPlugin(options) {
|
|
|
1001
1024
|
}
|
|
1002
1025
|
}
|
|
1003
1026
|
// Vector adapters (pgvector) — only when __vector is injected
|
|
1004
|
-
if (vector && vectorAdapterColumns.length > 0) {
|
|
1027
|
+
if ((vector || (perColumn && Object.keys(perColumn).length > 0)) && vectorAdapterColumns.length > 0) {
|
|
1005
1028
|
for (const { adapter, columns } of vectorAdapterColumns) {
|
|
1006
1029
|
for (const column of columns) {
|
|
1007
|
-
|
|
1030
|
+
// One query vector, embedded by the text model: a column of
|
|
1031
|
+
// another declared length (an image-embedding column beside
|
|
1032
|
+
// the text one) holds vectors it cannot be compared with —
|
|
1033
|
+
// pgvector raises on the operator — so it sits this search out.
|
|
1034
|
+
// Each column is searched with the vector its own model produced.
|
|
1035
|
+
const query = perColumn ? perColumn[column.attributeName] : (vector ? { vector, model } : null);
|
|
1036
|
+
if (!query)
|
|
1037
|
+
continue;
|
|
1038
|
+
const declared = column.adapterData?.dimensions;
|
|
1039
|
+
if (declared && declared !== query.vector.length)
|
|
1040
|
+
continue;
|
|
1041
|
+
const applyFor = (alias) => adapter.buildFilterApply(sql, alias, column, { vector: query.vector, model: query.model, metric: 'COSINE' }, build);
|
|
1008
1042
|
const result = applyFor($condition.alias);
|
|
1009
1043
|
if (!result)
|
|
1010
1044
|
continue;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "graphile-search",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.20.0",
|
|
4
4
|
"description": "Unified PostGraphile v5 search plugin — abstracts tsvector, BM25, pg_trgm, and pgvector behind a single adapter-based architecture with composite searchScore",
|
|
5
5
|
"author": "Constructive <developers@constructive.io>",
|
|
6
6
|
"homepage": "https://github.com/constructive-io/constructive",
|
|
@@ -65,5 +65,5 @@
|
|
|
65
65
|
"hybrid-search",
|
|
66
66
|
"searchScore"
|
|
67
67
|
],
|
|
68
|
-
"gitHead": "
|
|
68
|
+
"gitHead": "7e76a11316927407d054c3c4b3f6cfbfeae4440a"
|
|
69
69
|
}
|
package/plugin.d.ts
CHANGED
|
@@ -22,6 +22,13 @@ import 'graphile-connection-filter';
|
|
|
22
22
|
import type { PgCodecWithAttributes } from '@dataplan/pg';
|
|
23
23
|
import type { GraphileConfig } from 'graphile-config';
|
|
24
24
|
import type { UnifiedSearchOptions } from './types';
|
|
25
|
+
/**
|
|
26
|
+
* Carried on a filter field that searches vector columns, for whatever embeds
|
|
27
|
+
* the search text (graphile-llm): which columns, and the model each column's
|
|
28
|
+
* own definition names. Vectors of two models are not comparable, so the text
|
|
29
|
+
* is embedded per model.
|
|
30
|
+
*/
|
|
31
|
+
export declare const VECTOR_COLUMNS_EXT = "graphileSearchVectorColumns";
|
|
25
32
|
declare global {
|
|
26
33
|
namespace GraphileBuild {
|
|
27
34
|
interface Inflection {
|
package/plugin.js
CHANGED
|
@@ -18,12 +18,24 @@
|
|
|
18
18
|
* Uses the same Grafast meta system (setMeta/getMeta) as the individual plugins.
|
|
19
19
|
*/
|
|
20
20
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
21
|
+
exports.VECTOR_COLUMNS_EXT = void 0;
|
|
21
22
|
exports.createUnifiedSearchPlugin = createUnifiedSearchPlugin;
|
|
22
23
|
require("graphile-build");
|
|
23
24
|
require("graphile-build-pg");
|
|
24
25
|
require("graphile-connection-filter");
|
|
25
26
|
const pg_1 = require("@dataplan/pg");
|
|
26
27
|
const graphile_plugin_utils_1 = require("graphile-plugin-utils");
|
|
28
|
+
/**
|
|
29
|
+
* Carried on a filter field that searches vector columns, for whatever embeds
|
|
30
|
+
* the search text (graphile-llm): which columns, and the model each column's
|
|
31
|
+
* own definition names. Vectors of two models are not comparable, so the text
|
|
32
|
+
* is embedded per model.
|
|
33
|
+
*/
|
|
34
|
+
exports.VECTOR_COLUMNS_EXT = 'graphileSearchVectorColumns';
|
|
35
|
+
function vectorColumnRef(column) {
|
|
36
|
+
const model = column.adapterData?.model;
|
|
37
|
+
return { attributeName: column.attributeName, ...(model && { model }) };
|
|
38
|
+
}
|
|
27
39
|
/**
|
|
28
40
|
* Read the @searchConfig smart tag from a codec's extensions.
|
|
29
41
|
* Returns undefined if no searchConfig tag is present.
|
|
@@ -924,6 +936,9 @@ function createUnifiedSearchPlugin(options) {
|
|
|
924
936
|
}, {
|
|
925
937
|
description: build.wrapDescription(`${adapter.name.toUpperCase()} search on the \`${column.attributeName}\` column.`, 'field'),
|
|
926
938
|
type: build.getTypeByName(adapter.getFilterTypeName(build)),
|
|
939
|
+
...(adapter.name === 'vector' && {
|
|
940
|
+
extensions: { [exports.VECTOR_COLUMNS_EXT]: [vectorColumnRef(column)] },
|
|
941
|
+
}),
|
|
927
942
|
apply: function plan($condition, val) {
|
|
928
943
|
if (val == null)
|
|
929
944
|
return;
|
|
@@ -970,15 +985,24 @@ function createUnifiedSearchPlugin(options) {
|
|
|
970
985
|
'When the LLM plugin is active, pgvector also participates via auto-embedding. ' +
|
|
971
986
|
'Rows matching ANY algorithm are returned. All matching score fields are populated.', 'field'),
|
|
972
987
|
type: build.graphql.GraphQLString,
|
|
988
|
+
extensions: {
|
|
989
|
+
[exports.VECTOR_COLUMNS_EXT]: vectorAdapterColumns.flatMap((ac) => ac.columns.map(vectorColumnRef)),
|
|
990
|
+
},
|
|
973
991
|
apply: function plan($condition, val) {
|
|
974
992
|
if (val == null)
|
|
975
993
|
return;
|
|
976
994
|
// Support both plain string and { __text, __vector } from LLM plugin
|
|
977
995
|
let text;
|
|
978
996
|
let vector = null;
|
|
997
|
+
let model;
|
|
998
|
+
// Absent: one vector for every column. Present: a column it does
|
|
999
|
+
// not name has no vector of its own model and sits this search out.
|
|
1000
|
+
let perColumn;
|
|
979
1001
|
if (typeof val === 'object' && val.__text) {
|
|
980
1002
|
text = val.__text;
|
|
981
1003
|
vector = val.__vector ?? null;
|
|
1004
|
+
model = val.__model ?? undefined;
|
|
1005
|
+
perColumn = val.__vectors ?? undefined;
|
|
982
1006
|
}
|
|
983
1007
|
else {
|
|
984
1008
|
text = typeof val === 'string' ? val : String(val);
|
|
@@ -1004,10 +1028,21 @@ function createUnifiedSearchPlugin(options) {
|
|
|
1004
1028
|
}
|
|
1005
1029
|
}
|
|
1006
1030
|
// Vector adapters (pgvector) — only when __vector is injected
|
|
1007
|
-
if (vector && vectorAdapterColumns.length > 0) {
|
|
1031
|
+
if ((vector || (perColumn && Object.keys(perColumn).length > 0)) && vectorAdapterColumns.length > 0) {
|
|
1008
1032
|
for (const { adapter, columns } of vectorAdapterColumns) {
|
|
1009
1033
|
for (const column of columns) {
|
|
1010
|
-
|
|
1034
|
+
// One query vector, embedded by the text model: a column of
|
|
1035
|
+
// another declared length (an image-embedding column beside
|
|
1036
|
+
// the text one) holds vectors it cannot be compared with —
|
|
1037
|
+
// pgvector raises on the operator — so it sits this search out.
|
|
1038
|
+
// Each column is searched with the vector its own model produced.
|
|
1039
|
+
const query = perColumn ? perColumn[column.attributeName] : (vector ? { vector, model } : null);
|
|
1040
|
+
if (!query)
|
|
1041
|
+
continue;
|
|
1042
|
+
const declared = column.adapterData?.dimensions;
|
|
1043
|
+
if (declared && declared !== query.vector.length)
|
|
1044
|
+
continue;
|
|
1045
|
+
const applyFor = (alias) => adapter.buildFilterApply(sql, alias, column, { vector: query.vector, model: query.model, metric: 'COSINE' }, build);
|
|
1011
1046
|
const result = applyFor($condition.alias);
|
|
1012
1047
|
if (!result)
|
|
1013
1048
|
continue;
|