arocapi 5.0.0-beta.4 → 5.0.0-beta.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +1 -1
- package/README.md +3 -3
- package/dist/app.d.ts +19 -13
- package/dist/app.js +13 -10
- package/dist/app.js.map +1 -1
- package/dist/routes/capabilities.d.ts +2 -2
- package/dist/routes/capabilities.js +2 -2
- package/dist/routes/capabilities.js.map +1 -1
- package/dist/routes/entities.js +8 -9
- package/dist/routes/entities.js.map +1 -1
- package/dist/routes/file.js +5 -1
- package/dist/routes/file.js.map +1 -1
- package/dist/routes/search.d.ts +2 -4
- package/dist/routes/search.js +75 -97
- package/dist/routes/search.js.map +1 -1
- package/dist/search/advanced.d.ts +8 -0
- package/dist/search/advanced.js +198 -0
- package/dist/search/advanced.js.map +1 -0
- package/dist/search/backend.d.ts +95 -0
- package/dist/search/backend.js +5 -0
- package/dist/search/backend.js.map +1 -0
- package/dist/search/fields.d.ts +63 -0
- package/dist/search/fields.js +203 -0
- package/dist/search/fields.js.map +1 -0
- package/dist/search/opensearch/advanced.d.ts +4 -0
- package/dist/search/opensearch/advanced.js +28 -0
- package/dist/search/opensearch/advanced.js.map +1 -0
- package/dist/search/opensearch/opensearch.d.ts +16 -0
- package/dist/search/opensearch/opensearch.js +187 -0
- package/dist/search/opensearch/opensearch.js.map +1 -0
- package/dist/search/opensearch/visibility.d.ts +3 -0
- package/dist/search/opensearch/visibility.js +27 -0
- package/dist/search/opensearch/visibility.js.map +1 -0
- package/dist/search/postgres/advanced.d.ts +7 -0
- package/dist/search/postgres/advanced.js +54 -0
- package/dist/search/postgres/advanced.js.map +1 -0
- package/dist/search/postgres/facets.d.ts +2 -0
- package/dist/search/postgres/facets.js +27 -0
- package/dist/search/postgres/facets.js.map +1 -0
- package/dist/search/postgres/geo.d.ts +6 -0
- package/dist/search/postgres/geo.js +42 -0
- package/dist/search/postgres/geo.js.map +1 -0
- package/dist/search/postgres/postgres.d.ts +19 -0
- package/dist/search/postgres/postgres.js +157 -0
- package/dist/search/postgres/postgres.js.map +1 -0
- package/dist/search/postgres/values.d.ts +8 -0
- package/dist/search/postgres/values.js +20 -0
- package/dist/search/postgres/values.js.map +1 -0
- package/dist/search/postgres/visibility.d.ts +4 -0
- package/dist/search/postgres/visibility.js +38 -0
- package/dist/search/postgres/visibility.js.map +1 -0
- package/dist/search/requestFilters.d.ts +39 -0
- package/dist/search/requestFilters.js +125 -0
- package/dist/search/requestFilters.js.map +1 -0
- package/dist/search/visibility.d.ts +51 -0
- package/dist/search/visibility.js +100 -0
- package/dist/search/visibility.js.map +1 -0
- package/dist/types/visibility.d.ts +3 -2
- package/dist/utils/visibility.js +1 -1
- package/dist/utils/visibility.js.map +1 -1
- package/package.json +46 -26
- package/prisma/models/entity.prisma +9 -8
- package/prisma/models/entitySearch.prisma +21 -0
- package/prisma/models/file.prisma +3 -3
- package/src/app.ts +53 -24
- package/src/routes/capabilities.ts +4 -4
- package/src/routes/entities.ts +10 -9
- package/src/routes/file.ts +5 -1
- package/src/routes/search.ts +104 -118
- package/src/search/advanced.ts +258 -0
- package/src/search/backend.ts +91 -0
- package/src/search/fields.ts +320 -0
- package/src/search/opensearch/advanced.ts +36 -0
- package/src/search/opensearch/opensearch.ts +290 -0
- package/src/search/opensearch/visibility.ts +38 -0
- package/src/search/postgres/advanced.ts +76 -0
- package/src/search/postgres/facets.ts +34 -0
- package/src/search/postgres/geo.ts +58 -0
- package/src/search/postgres/postgres.ts +232 -0
- package/src/search/postgres/values.ts +33 -0
- package/src/search/postgres/visibility.ts +54 -0
- package/src/search/requestFilters.ts +178 -0
- package/src/search/visibility.ts +193 -0
- package/src/types/visibility.ts +6 -3
- package/src/utils/visibility.ts +1 -1
- package/dist/app.test.d.ts +0 -1
- package/dist/app.test.js +0 -67
- package/dist/app.test.js.map +0 -1
- package/dist/errorHandling.test.d.ts +0 -1
- package/dist/errorHandling.test.js +0 -222
- package/dist/errorHandling.test.js.map +0 -1
- package/dist/index.dev.d.ts +0 -1
- package/dist/index.dev.js +0 -157
- package/dist/index.dev.js.map +0 -1
- package/dist/routes/capabilities.test.d.ts +0 -1
- package/dist/routes/capabilities.test.js +0 -81
- package/dist/routes/capabilities.test.js.map +0 -1
- package/dist/routes/crate.test.d.ts +0 -1
- package/dist/routes/crate.test.js +0 -390
- package/dist/routes/crate.test.js.map +0 -1
- package/dist/routes/entities.test.d.ts +0 -1
- package/dist/routes/entities.test.js +0 -277
- package/dist/routes/entities.test.js.map +0 -1
- package/dist/routes/entity.test.d.ts +0 -1
- package/dist/routes/entity.test.js +0 -136
- package/dist/routes/entity.test.js.map +0 -1
- package/dist/routes/file.test.d.ts +0 -1
- package/dist/routes/file.test.js +0 -425
- package/dist/routes/file.test.js.map +0 -1
- package/dist/routes/files.test.d.ts +0 -1
- package/dist/routes/files.test.js +0 -245
- package/dist/routes/files.test.js.map +0 -1
- package/dist/routes/search.test.d.ts +0 -1
- package/dist/routes/search.test.js +0 -833
- package/dist/routes/search.test.js.map +0 -1
- package/dist/test/helpers/fastify.d.ts +0 -9
- package/dist/test/helpers/fastify.js +0 -17
- package/dist/test/helpers/fastify.js.map +0 -1
- package/dist/test/integration.setup.d.ts +0 -47
- package/dist/test/integration.setup.js +0 -195
- package/dist/test/integration.setup.js.map +0 -1
- package/dist/test/integration.test.d.ts +0 -1
- package/dist/test/integration.test.js +0 -762
- package/dist/test/integration.test.js.map +0 -1
- package/dist/transformers/default.test.d.ts +0 -1
- package/dist/transformers/default.test.js +0 -225
- package/dist/transformers/default.test.js.map +0 -1
- package/dist/transformers/transformer.test.d.ts +0 -1
- package/dist/transformers/transformer.test.js +0 -165
- package/dist/transformers/transformer.test.js.map +0 -1
- package/dist/utils/errors.test.d.ts +0 -1
- package/dist/utils/errors.test.js +0 -79
- package/dist/utils/errors.test.js.map +0 -1
- package/dist/utils/queryBuilder.d.ts +0 -47
- package/dist/utils/queryBuilder.js +0 -107
- package/dist/utils/queryBuilder.js.map +0 -1
- package/prisma.config.ts +0 -9
- package/src/app.test.ts +0 -91
- package/src/errorHandling.test.ts +0 -288
- package/src/index.dev.ts +0 -202
- package/src/reset.d.ts +0 -1
- package/src/routes/__snapshots__/capabilities.test.ts.snap +0 -34
- package/src/routes/__snapshots__/entities.test.ts.snap +0 -96
- package/src/routes/__snapshots__/entity.test.ts.snap +0 -74
- package/src/routes/__snapshots__/search.test.ts.snap +0 -165
- package/src/routes/capabilities.test.ts +0 -102
- package/src/routes/crate.test.ts +0 -484
- package/src/routes/entities.test.ts +0 -324
- package/src/routes/entity.test.ts +0 -166
- package/src/routes/file.test.ts +0 -526
- package/src/routes/files.test.ts +0 -304
- package/src/routes/search.test.ts +0 -959
- package/src/test/__snapshots__/integration.test.ts.snap +0 -18
- package/src/test/helpers/fastify.ts +0 -26
- package/src/test/integration.setup.ts +0 -220
- package/src/test/integration.test.ts +0 -953
- package/src/transformers/default.test.ts +0 -276
- package/src/transformers/transformer.test.ts +0 -226
- package/src/utils/errors.test.ts +0 -90
- package/src/utils/queryBuilder.ts +0 -157
|
@@ -0,0 +1,290 @@
|
|
|
1
|
+
import { isDeepStrictEqual } from 'node:util';
|
|
2
|
+
import type { Client } from '@opensearch-project/opensearch';
|
|
3
|
+
import type { SortOrder } from '@opensearch-project/opensearch/api/_types/_common.js';
|
|
4
|
+
import type { QueryContainer } from '@opensearch-project/opensearch/api/_types/_common.query_dsl.js';
|
|
5
|
+
import type { Search_RequestBody, Search_Response } from '@opensearch-project/opensearch/api/index.js';
|
|
6
|
+
import {
|
|
7
|
+
type BoundingBox,
|
|
8
|
+
type FacetBucket,
|
|
9
|
+
GEOHASH_GRID_CELLS,
|
|
10
|
+
SearchQueryError,
|
|
11
|
+
type SearchRequest,
|
|
12
|
+
type SearchResult,
|
|
13
|
+
} from '../backend.js';
|
|
14
|
+
import {
|
|
15
|
+
capabilitiesFromFields,
|
|
16
|
+
type FieldDeclaration,
|
|
17
|
+
type FieldDeclarations,
|
|
18
|
+
isBasic,
|
|
19
|
+
resolveFields,
|
|
20
|
+
type SearchDocument,
|
|
21
|
+
storeDocuments,
|
|
22
|
+
} from '../fields.js';
|
|
23
|
+
import { filtersCondition } from '../requestFilters.js';
|
|
24
|
+
import { checkSearchVisibility } from '../visibility.js';
|
|
25
|
+
import { advancedQueryFor } from './advanced.js';
|
|
26
|
+
import { visibilityQuery } from './visibility.js';
|
|
27
|
+
|
|
28
|
+
export type OpenSearchBackendOptions<Fields extends FieldDeclarations> = {
|
|
29
|
+
client: Client;
|
|
30
|
+
fields: Fields;
|
|
31
|
+
/** Default `entities` */
|
|
32
|
+
indexName?: string;
|
|
33
|
+
/** Passed to each bulk `index` and `remove`. Default `false` */
|
|
34
|
+
refresh?: boolean | 'wait_for';
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
// A keyword term is limited to 32766 bytes, up to 4 per character
|
|
38
|
+
const EXACT_FORM = { exact: { type: 'keyword', ignore_above: 8191 } };
|
|
39
|
+
|
|
40
|
+
const mappingFor = (declaration: FieldDeclaration) => {
|
|
41
|
+
const { type } = declaration;
|
|
42
|
+
switch (type) {
|
|
43
|
+
case 'text':
|
|
44
|
+
return { type, fields: EXACT_FORM };
|
|
45
|
+
case 'number':
|
|
46
|
+
return { type: 'double' };
|
|
47
|
+
case 'geo':
|
|
48
|
+
return { type: 'geo_point' };
|
|
49
|
+
default:
|
|
50
|
+
return { type };
|
|
51
|
+
}
|
|
52
|
+
};
|
|
53
|
+
|
|
54
|
+
const FACET_ORDER: Record<string, SortOrder>[] = [{ _count: 'desc' }, { _key: 'asc' }];
|
|
55
|
+
|
|
56
|
+
type TermsBucket = { key: string | number; key_as_string?: string; doc_count: number };
|
|
57
|
+
|
|
58
|
+
// Not a valid field name, so it can't collide with a facet
|
|
59
|
+
const GEOHASH_GRID = 'geohash-grid';
|
|
60
|
+
|
|
61
|
+
const geoBox = ({ topRight, bottomLeft }: BoundingBox) => ({
|
|
62
|
+
top_left: { lat: topRight.lat, lon: bottomLeft.lng },
|
|
63
|
+
bottom_right: { lat: bottomLeft.lat, lon: topRight.lng },
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
type BulkItem = Record<
|
|
67
|
+
string,
|
|
68
|
+
{ _id?: string | null; status: number; error?: { type: string; reason?: string | null } }
|
|
69
|
+
>;
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* OpenSearch search backend: holds one search document per entity in an index built from the field declarations
|
|
73
|
+
*/
|
|
74
|
+
export const OpenSearchBackend = <const Fields extends FieldDeclarations>({
|
|
75
|
+
client,
|
|
76
|
+
fields,
|
|
77
|
+
indexName = 'entities',
|
|
78
|
+
refresh = false,
|
|
79
|
+
}: OpenSearchBackendOptions<Fields>) => {
|
|
80
|
+
const capabilities = capabilitiesFromFields(fields);
|
|
81
|
+
const { basic, facets, geoField } = resolveFields(fields);
|
|
82
|
+
|
|
83
|
+
const exactField = (name: string) => (fields[name]?.type === 'text' ? `${name}.exact` : name);
|
|
84
|
+
const basicFields = basic.map(({ name, weight }) => `${name}^${weight}`);
|
|
85
|
+
const highlight: Search_RequestBody['highlight'] = {
|
|
86
|
+
fields: Object.fromEntries(
|
|
87
|
+
Object.entries(fields)
|
|
88
|
+
.filter(
|
|
89
|
+
([, declaration]) =>
|
|
90
|
+
['text', 'keyword'].includes(declaration.type) && (isBasic(declaration) || declaration.advanced),
|
|
91
|
+
)
|
|
92
|
+
.map(([name]) => [name, {}]),
|
|
93
|
+
),
|
|
94
|
+
};
|
|
95
|
+
|
|
96
|
+
const queryFor = (request: SearchRequest): QueryContainer => {
|
|
97
|
+
if (request.searchType === 'advanced') {
|
|
98
|
+
return advancedQueryFor(request.query, fields);
|
|
99
|
+
}
|
|
100
|
+
if (request.query.trim() === '') {
|
|
101
|
+
return { match_all: {} };
|
|
102
|
+
}
|
|
103
|
+
if (basicFields.length === 0) {
|
|
104
|
+
return { match_none: {} };
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
return {
|
|
108
|
+
multi_match: {
|
|
109
|
+
query: request.query,
|
|
110
|
+
fields: basicFields,
|
|
111
|
+
type: 'best_fields',
|
|
112
|
+
fuzziness: 'AUTO',
|
|
113
|
+
zero_terms_query: 'all',
|
|
114
|
+
},
|
|
115
|
+
};
|
|
116
|
+
};
|
|
117
|
+
|
|
118
|
+
const sortFor = ({ sort, order }: SearchRequest): Search_RequestBody['sort'] => [
|
|
119
|
+
sort === 'relevance' ? { _score: { order: 'desc' } } : { [exactField(sort)]: { order, missing: '_last' } },
|
|
120
|
+
{ id: { order: 'asc' } },
|
|
121
|
+
];
|
|
122
|
+
|
|
123
|
+
const aggregations: Search_RequestBody['aggs'] = Object.fromEntries(
|
|
124
|
+
facets.map(({ name, type, size }) => [
|
|
125
|
+
name,
|
|
126
|
+
type === 'date'
|
|
127
|
+
? {
|
|
128
|
+
date_histogram: {
|
|
129
|
+
field: name,
|
|
130
|
+
calendar_interval: 'year',
|
|
131
|
+
format: 'yyyy',
|
|
132
|
+
time_zone: 'UTC',
|
|
133
|
+
min_doc_count: 1,
|
|
134
|
+
order: { _key: 'asc' },
|
|
135
|
+
},
|
|
136
|
+
}
|
|
137
|
+
: {
|
|
138
|
+
terms: {
|
|
139
|
+
field: exactField(name),
|
|
140
|
+
size,
|
|
141
|
+
order: FACET_ORDER,
|
|
142
|
+
},
|
|
143
|
+
},
|
|
144
|
+
]),
|
|
145
|
+
);
|
|
146
|
+
|
|
147
|
+
const properties: Record<string, object> = {
|
|
148
|
+
id: { type: 'keyword' },
|
|
149
|
+
...Object.fromEntries(Object.entries(fields).map(([name, declaration]) => [name, mappingFor(declaration)])),
|
|
150
|
+
};
|
|
151
|
+
|
|
152
|
+
const bulk = async (operations: object[], tolerated: number[] = []) => {
|
|
153
|
+
if (operations.length === 0) {
|
|
154
|
+
return;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
const { body } = await client.bulk({ body: operations, refresh });
|
|
158
|
+
const failure = (body.items as BulkItem[])
|
|
159
|
+
.map((item) => Object.values(item)[0])
|
|
160
|
+
.find(({ error, status }) => error && !tolerated.includes(status));
|
|
161
|
+
if (failure?.error) {
|
|
162
|
+
const { _id, error } = failure;
|
|
163
|
+
throw new Error(`Search document "${_id}" failed in index "${indexName}": ${error.type}: ${error.reason}`);
|
|
164
|
+
}
|
|
165
|
+
};
|
|
166
|
+
|
|
167
|
+
return {
|
|
168
|
+
capabilities,
|
|
169
|
+
|
|
170
|
+
search: async (request: SearchRequest): Promise<SearchResult> => {
|
|
171
|
+
const { boundingBox, geohashPrecision } = request;
|
|
172
|
+
const filter: QueryContainer[] = [
|
|
173
|
+
visibilityQuery(filtersCondition(request.filters), exactField),
|
|
174
|
+
...(geoField && boundingBox ? [{ geo_bounding_box: { [geoField]: geoBox(boundingBox) } }] : []),
|
|
175
|
+
visibilityQuery(checkSearchVisibility(fields, request.visibility), exactField),
|
|
176
|
+
];
|
|
177
|
+
const geohashGrid: Search_RequestBody['aggs'] =
|
|
178
|
+
geoField && geohashPrecision !== undefined
|
|
179
|
+
? {
|
|
180
|
+
[GEOHASH_GRID]: {
|
|
181
|
+
geohash_grid: { field: geoField, precision: geohashPrecision, size: GEOHASH_GRID_CELLS },
|
|
182
|
+
},
|
|
183
|
+
}
|
|
184
|
+
: {};
|
|
185
|
+
|
|
186
|
+
let response: Search_Response;
|
|
187
|
+
try {
|
|
188
|
+
response = await client.search({
|
|
189
|
+
index: indexName,
|
|
190
|
+
body: {
|
|
191
|
+
query: { bool: { must: [queryFor(request)], filter } },
|
|
192
|
+
aggs: { ...aggregations, ...geohashGrid },
|
|
193
|
+
highlight,
|
|
194
|
+
sort: sortFor(request),
|
|
195
|
+
from: request.offset,
|
|
196
|
+
size: request.limit,
|
|
197
|
+
track_total_hits: true,
|
|
198
|
+
_source: false,
|
|
199
|
+
},
|
|
200
|
+
});
|
|
201
|
+
} catch (error) {
|
|
202
|
+
const { statusCode, message } = error as Error & { statusCode?: number };
|
|
203
|
+
if (statusCode === 400) {
|
|
204
|
+
throw new SearchQueryError(message);
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
throw error;
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
const { hits, aggregations: buckets = {} } = response.body;
|
|
211
|
+
|
|
212
|
+
return {
|
|
213
|
+
total: (hits.total as { value: number }).value,
|
|
214
|
+
hits: hits.hits.map(({ _id, _score, highlight: highlighted }) => {
|
|
215
|
+
const searchExtra = {
|
|
216
|
+
...(typeof _score === 'number' && { score: _score }),
|
|
217
|
+
...(highlighted && Object.keys(highlighted).length > 0 && { highlight: highlighted }),
|
|
218
|
+
};
|
|
219
|
+
|
|
220
|
+
return { id: _id as string, ...(Object.keys(searchExtra).length > 0 && { searchExtra }) };
|
|
221
|
+
}),
|
|
222
|
+
facets: Object.fromEntries(
|
|
223
|
+
facets.map(({ name }) => [
|
|
224
|
+
name,
|
|
225
|
+
(buckets[name] as { buckets: TermsBucket[] }).buckets.map(
|
|
226
|
+
({ key, key_as_string, doc_count }): FacetBucket => ({
|
|
227
|
+
name: key_as_string ?? String(key),
|
|
228
|
+
count: doc_count,
|
|
229
|
+
}),
|
|
230
|
+
),
|
|
231
|
+
]),
|
|
232
|
+
),
|
|
233
|
+
...(GEOHASH_GRID in buckets && {
|
|
234
|
+
geohashGrid: Object.fromEntries(
|
|
235
|
+
(buckets[GEOHASH_GRID] as { buckets: TermsBucket[] }).buckets.map(({ key, doc_count }) => [key, doc_count]),
|
|
236
|
+
),
|
|
237
|
+
}),
|
|
238
|
+
};
|
|
239
|
+
},
|
|
240
|
+
|
|
241
|
+
/**
|
|
242
|
+
* Create the index with a mapping built from the field declarations
|
|
243
|
+
* Does nothing if it exists with that mapping, and throws, naming the field, if it exists with another
|
|
244
|
+
*/
|
|
245
|
+
createIndex: async () => {
|
|
246
|
+
const { body: exists } = await client.indices.exists({ index: indexName });
|
|
247
|
+
if (!exists) {
|
|
248
|
+
await client.indices.create({ index: indexName, body: { mappings: { dynamic: 'strict', properties } } });
|
|
249
|
+
|
|
250
|
+
return;
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
const { body } = await client.indices.getMapping({ index: indexName });
|
|
254
|
+
const existing = (body[indexName].mappings.properties ?? {}) as Record<string, unknown>;
|
|
255
|
+
const differing = [...new Set([...Object.keys(properties), ...Object.keys(existing)])].find(
|
|
256
|
+
(name) => !isDeepStrictEqual(properties[name], existing[name]),
|
|
257
|
+
);
|
|
258
|
+
if (differing) {
|
|
259
|
+
throw new Error(
|
|
260
|
+
`Index "${indexName}" exists with a different mapping for field "${differing}" than its declaration: ` +
|
|
261
|
+
`${JSON.stringify(existing[differing] ?? null)} instead of ${JSON.stringify(properties[differing] ?? null)}`,
|
|
262
|
+
);
|
|
263
|
+
}
|
|
264
|
+
},
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* Upsert search documents, replacing any already indexed with the same id
|
|
268
|
+
*/
|
|
269
|
+
index: async (docs: SearchDocument<Fields>[]) => {
|
|
270
|
+
const stored = storeDocuments(fields, docs);
|
|
271
|
+
|
|
272
|
+
await bulk(
|
|
273
|
+
stored.flatMap(({ id, values, point }) => [
|
|
274
|
+
{ index: { _index: indexName, _id: id } },
|
|
275
|
+
{ id, ...values, ...(geoField && point && { [geoField]: { lat: point.lat, lon: point.lng } }) },
|
|
276
|
+
]),
|
|
277
|
+
);
|
|
278
|
+
},
|
|
279
|
+
|
|
280
|
+
/**
|
|
281
|
+
* Remove search documents, ignoring ids that aren't indexed
|
|
282
|
+
*/
|
|
283
|
+
remove: async (ids: string[]) => {
|
|
284
|
+
await bulk(
|
|
285
|
+
ids.map((id) => ({ delete: { _index: indexName, _id: id } })),
|
|
286
|
+
[404],
|
|
287
|
+
);
|
|
288
|
+
},
|
|
289
|
+
};
|
|
290
|
+
};
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
import type { QueryContainer } from '@opensearch-project/opensearch/api/_types/_common.query_dsl.js';
|
|
2
|
+
import { toWire } from '../fields.js';
|
|
3
|
+
import type { VisibilityCondition } from '../visibility.js';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* Translate a checked search visibility filter to an OpenSearch filter, comparing each field's exact form
|
|
7
|
+
*/
|
|
8
|
+
export const visibilityQuery = (
|
|
9
|
+
condition: VisibilityCondition,
|
|
10
|
+
exactField: (name: string) => string,
|
|
11
|
+
): QueryContainer => {
|
|
12
|
+
const translate = (node: VisibilityCondition): QueryContainer => {
|
|
13
|
+
switch (node.type) {
|
|
14
|
+
case 'and':
|
|
15
|
+
return { bool: { filter: node.clauses.map(translate) } };
|
|
16
|
+
case 'or':
|
|
17
|
+
return node.clauses.length === 0
|
|
18
|
+
? { match_none: {} }
|
|
19
|
+
: { bool: { should: node.clauses.map(translate), minimum_should_match: 1 } };
|
|
20
|
+
case 'not':
|
|
21
|
+
return { bool: { must_not: [translate(node.clause)] } };
|
|
22
|
+
case 'in':
|
|
23
|
+
return { terms: { [exactField(node.field)]: node.values.map(toWire) } };
|
|
24
|
+
case 'present':
|
|
25
|
+
return { exists: { field: node.field } };
|
|
26
|
+
case 'range':
|
|
27
|
+
return {
|
|
28
|
+
range: {
|
|
29
|
+
[node.field]: Object.fromEntries(
|
|
30
|
+
Object.entries(node.bounds).map(([bound, value]) => [bound, toWire(value)]),
|
|
31
|
+
),
|
|
32
|
+
},
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
return translate(condition);
|
|
38
|
+
};
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { join, type Sql, sqltag as sql } from '@prisma/client/runtime/client';
|
|
2
|
+
import { matchedDateSpan } from '../advanced.js';
|
|
3
|
+
import { type AdvancedQuery, SearchQueryError } from '../backend.js';
|
|
4
|
+
import type { FieldDeclarations, FieldType } from '../fields.js';
|
|
5
|
+
import { fieldValue, parameter, TEXT_SEARCH_CONFIG, valuesOf } from './values.js';
|
|
6
|
+
|
|
7
|
+
// A tsquery's lexemes as text: single-quoted, with quotes doubled
|
|
8
|
+
const LEXEME = "'((?:[^']|'')*)'";
|
|
9
|
+
|
|
10
|
+
const exactValue = (field: string, type: string, value: string) => {
|
|
11
|
+
if (type === 'boolean' && value !== 'true' && value !== 'false') {
|
|
12
|
+
throw new SearchQueryError(`"${field}" is a boolean field: "${value}" isn't true or false`);
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
return type === 'boolean' ? value === 'true' : value;
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
const exactCondition = (field: string, type: FieldType, value: string) => {
|
|
19
|
+
if (type === 'date') {
|
|
20
|
+
const { first, last } = matchedDateSpan(field, value);
|
|
21
|
+
|
|
22
|
+
return sql`${fieldValue(type)} BETWEEN ${parameter(type, first)} AND ${parameter(type, last)}`;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
return sql`${fieldValue(type)} = ${parameter(type, exactValue(field, type, value))}`;
|
|
26
|
+
};
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Translate an advanced query tree to a SQL condition on the `search` row, and a rank over its positive text matches
|
|
30
|
+
* Text values match on the `text` tsvector, whose lexemes are prefixed `<field>.`
|
|
31
|
+
*/
|
|
32
|
+
export const advancedQueryFor = (query: AdvancedQuery, fields: FieldDeclarations) => {
|
|
33
|
+
const ranked: Sql[] = [];
|
|
34
|
+
|
|
35
|
+
const tsquery = (field: string, value: string, phrase: boolean) => {
|
|
36
|
+
const parsed = phrase
|
|
37
|
+
? sql`phraseto_tsquery(${TEXT_SEARCH_CONFIG}, ${value}::text)`
|
|
38
|
+
: sql`plainto_tsquery(${TEXT_SEARCH_CONFIG}, ${value}::text)`;
|
|
39
|
+
|
|
40
|
+
return sql`regexp_replace(${parsed}::text, ${LEXEME}::text, ${`'${field}.\\1'`}::text, 'g')::tsquery`;
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
const translate = (node: AdvancedQuery, negated: boolean): Sql => {
|
|
44
|
+
switch (node.type) {
|
|
45
|
+
case 'and':
|
|
46
|
+
case 'or':
|
|
47
|
+
return sql`(${join(
|
|
48
|
+
node.clauses.map((clause) => translate(clause, negated)),
|
|
49
|
+
node.type === 'and' ? ' AND ' : ' OR ',
|
|
50
|
+
)})`;
|
|
51
|
+
case 'not':
|
|
52
|
+
return sql`NOT ${translate(node.clause, !negated)}`;
|
|
53
|
+
case 'match': {
|
|
54
|
+
const { field, value, phrase } = node;
|
|
55
|
+
const { type } = fields[field];
|
|
56
|
+
if (type === 'text') {
|
|
57
|
+
const matching = tsquery(field, value, phrase);
|
|
58
|
+
if (!negated) {
|
|
59
|
+
ranked.push(matching);
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
return sql`search.text @@ ${matching}`;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
return sql`EXISTS (SELECT FROM ${valuesOf(field)} WHERE ${exactCondition(field, type, value)})`;
|
|
66
|
+
}
|
|
67
|
+
default:
|
|
68
|
+
return sql`TRUE`;
|
|
69
|
+
}
|
|
70
|
+
};
|
|
71
|
+
|
|
72
|
+
const condition = translate(query, false);
|
|
73
|
+
const rank = ranked.length > 0 ? sql`ts_rank(search.text, ${join(ranked, ' || ')})` : sql`0`;
|
|
74
|
+
|
|
75
|
+
return { condition, rank };
|
|
76
|
+
};
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
import { sqltag as sql } from '@prisma/client/runtime/client';
|
|
2
|
+
import { fieldValue, valuesOf } from './values.js';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* A facet's buckets over the `matched` rows: the `size` commonest values, most matches first
|
|
6
|
+
*/
|
|
7
|
+
export const valueBuckets = (name: string, size: number) => sql`(
|
|
8
|
+
SELECT coalesce(jsonb_agg(
|
|
9
|
+
jsonb_build_object('name', bucket.name, 'count', bucket.count) ORDER BY bucket.count DESC, bucket.name COLLATE "C"
|
|
10
|
+
), '[]')
|
|
11
|
+
FROM (
|
|
12
|
+
SELECT value.name, count(*)::int AS count
|
|
13
|
+
FROM matched AS search, LATERAL (SELECT DISTINCT field.value AS name FROM ${valuesOf(name)}) AS value
|
|
14
|
+
GROUP BY value.name
|
|
15
|
+
ORDER BY count DESC, value.name COLLATE "C"
|
|
16
|
+
LIMIT ${size}::int
|
|
17
|
+
) AS bucket
|
|
18
|
+
)`;
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* A date facet's buckets over the `matched` rows: one per UTC year with matches, named `YYYY`, in year order
|
|
22
|
+
*/
|
|
23
|
+
export const yearBuckets = (name: string) => sql`(
|
|
24
|
+
SELECT coalesce(jsonb_agg(
|
|
25
|
+
jsonb_build_object('name', to_char(bucket.year, 'YYYY'), 'count', bucket.count) ORDER BY bucket.year
|
|
26
|
+
), '[]')
|
|
27
|
+
FROM (
|
|
28
|
+
SELECT value.year, count(*)::int AS count
|
|
29
|
+
FROM matched AS search, LATERAL (
|
|
30
|
+
SELECT DISTINCT date_trunc('year', ${fieldValue('date')} AT TIME ZONE 'UTC') AS year FROM ${valuesOf(name)}
|
|
31
|
+
) AS value
|
|
32
|
+
GROUP BY value.year
|
|
33
|
+
) AS bucket
|
|
34
|
+
)`;
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { type Sql, sqltag as sql } from '@prisma/client/runtime/client';
|
|
2
|
+
import { type BoundingBox, GEOHASH_GRID_CELLS } from '../backend.js';
|
|
3
|
+
import type { GeoPoint } from '../fields.js';
|
|
4
|
+
|
|
5
|
+
const GEOHASH_ALPHABET = '0123456789bcdefghjkmnpqrstuvwxyz';
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* A point's 12-character geohash
|
|
9
|
+
*/
|
|
10
|
+
export const geohash = ({ lat, lng }: GeoPoint) => {
|
|
11
|
+
const ranges = [
|
|
12
|
+
[-180, 180],
|
|
13
|
+
[-90, 90],
|
|
14
|
+
];
|
|
15
|
+
let hash = '';
|
|
16
|
+
let bits = 0;
|
|
17
|
+
for (let bit = 0; bit < 60; bit++) {
|
|
18
|
+
const axis = bit % 2;
|
|
19
|
+
const range = ranges[axis];
|
|
20
|
+
const middle = (range[0] + range[1]) / 2;
|
|
21
|
+
const upper = (axis === 0 ? lng : lat) >= middle;
|
|
22
|
+
range[upper ? 0 : 1] = middle;
|
|
23
|
+
bits = bits * 2 + Number(upper);
|
|
24
|
+
if (bit % 5 === 4) {
|
|
25
|
+
hash += GEOHASH_ALPHABET[bits];
|
|
26
|
+
bits = 0;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
return hash;
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* A condition that the `search` row's point lies in the box, which may cross the antimeridian
|
|
35
|
+
*/
|
|
36
|
+
export const boundingBoxCondition = ({ topRight, bottomLeft }: BoundingBox): Sql => {
|
|
37
|
+
const box = (west: number, east: number) =>
|
|
38
|
+
sql`search.point <@ box(point(${west}::float8, ${bottomLeft.lat}::float8), point(${east}::float8, ${topRight.lat}::float8))`;
|
|
39
|
+
|
|
40
|
+
return bottomLeft.lng <= topRight.lng
|
|
41
|
+
? box(bottomLeft.lng, topRight.lng)
|
|
42
|
+
: sql`(${box(bottomLeft.lng, 180)} OR ${box(-180, topRight.lng)})`;
|
|
43
|
+
};
|
|
44
|
+
|
|
45
|
+
/**
|
|
46
|
+
* Counts of the `matched` rows with a point, per geohash cell of `precision` characters (at most `GEOHASH_GRID_CELLS`, those with the most)
|
|
47
|
+
*/
|
|
48
|
+
export const geohashGrid = (precision: number) => sql`(
|
|
49
|
+
SELECT coalesce(jsonb_object_agg(cell.geohash, cell.count), '{}')
|
|
50
|
+
FROM (
|
|
51
|
+
SELECT left(search.geohash, ${precision}::int) COLLATE "C" AS geohash, count(*)::int AS count
|
|
52
|
+
FROM matched AS search
|
|
53
|
+
WHERE search.geohash IS NOT NULL
|
|
54
|
+
GROUP BY 1
|
|
55
|
+
ORDER BY count DESC, geohash
|
|
56
|
+
LIMIT ${GEOHASH_GRID_CELLS}::int
|
|
57
|
+
) AS cell
|
|
58
|
+
)`;
|