arocapi 4.0.1 → 5.0.0-beta.10
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +1 -1
- package/README.md +37 -781
- package/dist/app.d.ts +38 -21
- package/dist/app.js +63 -45
- package/dist/app.js.map +1 -1
- package/dist/routes/capabilities.d.ts +7 -0
- package/dist/routes/capabilities.js +13 -0
- package/dist/routes/capabilities.js.map +1 -0
- package/dist/routes/entities.d.ts +5 -2
- package/dist/routes/entities.js +38 -58
- package/dist/routes/entities.js.map +1 -1
- package/dist/routes/entity.d.ts +7 -2
- package/dist/routes/entity.js +14 -30
- package/dist/routes/entity.js.map +1 -1
- package/dist/routes/file.d.ts +8 -2
- package/dist/routes/file.js +38 -73
- package/dist/routes/file.js.map +1 -1
- package/dist/routes/files.d.ts +5 -2
- package/dist/routes/files.js +25 -40
- package/dist/routes/files.js.map +1 -1
- package/dist/routes/metadata.d.ts +15 -0
- package/dist/routes/metadata.js +38 -0
- package/dist/routes/metadata.js.map +1 -0
- package/dist/routes/search.d.ts +8 -7
- package/dist/routes/search.js +91 -120
- package/dist/routes/search.js.map +1 -1
- package/dist/search/advanced.d.ts +8 -0
- package/dist/search/advanced.js +198 -0
- package/dist/search/advanced.js.map +1 -0
- package/dist/search/backend.d.ts +95 -0
- package/dist/search/backend.js +5 -0
- package/dist/search/backend.js.map +1 -0
- package/dist/search/fields.d.ts +63 -0
- package/dist/search/fields.js +203 -0
- package/dist/search/fields.js.map +1 -0
- package/dist/search/opensearch/advanced.d.ts +4 -0
- package/dist/search/opensearch/advanced.js +28 -0
- package/dist/search/opensearch/advanced.js.map +1 -0
- package/dist/search/opensearch/opensearch.d.ts +16 -0
- package/dist/search/opensearch/opensearch.js +187 -0
- package/dist/search/opensearch/opensearch.js.map +1 -0
- package/dist/search/opensearch/visibility.d.ts +3 -0
- package/dist/search/opensearch/visibility.js +27 -0
- package/dist/search/opensearch/visibility.js.map +1 -0
- package/dist/search/postgres/advanced.d.ts +7 -0
- package/dist/search/postgres/advanced.js +54 -0
- package/dist/search/postgres/advanced.js.map +1 -0
- package/dist/search/postgres/facets.d.ts +2 -0
- package/dist/search/postgres/facets.js +27 -0
- package/dist/search/postgres/facets.js.map +1 -0
- package/dist/search/postgres/geo.d.ts +6 -0
- package/dist/search/postgres/geo.js +42 -0
- package/dist/search/postgres/geo.js.map +1 -0
- package/dist/search/postgres/postgres.d.ts +19 -0
- package/dist/search/postgres/postgres.js +157 -0
- package/dist/search/postgres/postgres.js.map +1 -0
- package/dist/search/postgres/values.d.ts +8 -0
- package/dist/search/postgres/values.js +20 -0
- package/dist/search/postgres/values.js.map +1 -0
- package/dist/search/postgres/visibility.d.ts +4 -0
- package/dist/search/postgres/visibility.js +38 -0
- package/dist/search/postgres/visibility.js.map +1 -0
- package/dist/search/requestFilters.d.ts +39 -0
- package/dist/search/requestFilters.js +125 -0
- package/dist/search/requestFilters.js.map +1 -0
- package/dist/search/visibility.d.ts +51 -0
- package/dist/search/visibility.js +100 -0
- package/dist/search/visibility.js.map +1 -0
- package/dist/storage/backend.d.ts +58 -0
- package/dist/storage/backend.js +2 -0
- package/dist/storage/backend.js.map +1 -0
- package/dist/storage/deliver.d.ts +12 -0
- package/dist/storage/deliver.js +34 -0
- package/dist/storage/deliver.js.map +1 -0
- package/dist/storage/directory/directory.d.ts +6 -0
- package/dist/storage/directory/directory.js +24 -0
- package/dist/storage/directory/directory.js.map +1 -0
- package/dist/storage/entity-meta/entity-meta.d.ts +5 -0
- package/dist/storage/entity-meta/entity-meta.js +14 -0
- package/dist/storage/entity-meta/entity-meta.js.map +1 -0
- package/dist/storage/location.d.ts +19 -0
- package/dist/storage/location.js +36 -0
- package/dist/storage/location.js.map +1 -0
- package/dist/storage/ocfl/ocfl.d.ts +6 -0
- package/dist/storage/ocfl/ocfl.js +45 -0
- package/dist/storage/ocfl/ocfl.js.map +1 -0
- package/dist/storage/results.d.ts +7 -0
- package/dist/storage/results.js +22 -0
- package/dist/storage/results.js.map +1 -0
- package/dist/storage/s3/s3.d.ts +10 -0
- package/dist/storage/s3/s3.js +47 -0
- package/dist/storage/s3/s3.js.map +1 -0
- package/dist/transformers/default.d.ts +11 -22
- package/dist/transformers/default.js +41 -15
- package/dist/transformers/default.js.map +1 -1
- package/dist/types/access.d.ts +9 -0
- package/dist/types/access.js +2 -0
- package/dist/types/access.js.map +1 -0
- package/dist/types/context.d.ts +6 -0
- package/dist/types/context.js +2 -0
- package/dist/types/context.js.map +1 -0
- package/dist/types/transformers.d.ts +4 -10
- package/dist/types/visibility.d.ts +6 -0
- package/dist/types/visibility.js +2 -0
- package/dist/types/visibility.js.map +1 -0
- package/dist/utils/access.d.ts +17 -0
- package/dist/utils/access.js +41 -0
- package/dist/utils/access.js.map +1 -0
- package/dist/utils/errors.d.ts +19 -10
- package/dist/utils/errors.js +44 -8
- package/dist/utils/errors.js.map +1 -1
- package/dist/utils/headers.d.ts +1 -1
- package/dist/utils/headers.js +10 -1
- package/dist/utils/headers.js.map +1 -1
- package/dist/utils/validation.d.ts +2 -0
- package/dist/utils/validation.js +8 -0
- package/dist/utils/validation.js.map +1 -0
- package/dist/utils/visibility.d.ts +18 -0
- package/dist/utils/visibility.js +16 -0
- package/dist/utils/visibility.js.map +1 -0
- package/package.json +84 -45
- package/prisma/models/entity.prisma +9 -8
- package/prisma/models/entitySearch.prisma +21 -0
- package/prisma/models/file.prisma +3 -3
- package/src/app.ts +169 -102
- package/src/routes/capabilities.ts +22 -0
- package/src/routes/entities.ts +49 -68
- package/src/routes/entity.ts +22 -36
- package/src/routes/file.ts +54 -104
- package/src/routes/files.ts +32 -48
- package/src/routes/metadata.ts +67 -0
- package/src/routes/search.ts +132 -152
- package/src/search/advanced.ts +258 -0
- package/src/search/backend.ts +91 -0
- package/src/search/fields.ts +320 -0
- package/src/search/opensearch/advanced.ts +36 -0
- package/src/search/opensearch/opensearch.ts +290 -0
- package/src/search/opensearch/visibility.ts +38 -0
- package/src/search/postgres/advanced.ts +76 -0
- package/src/search/postgres/facets.ts +34 -0
- package/src/search/postgres/geo.ts +58 -0
- package/src/search/postgres/postgres.ts +232 -0
- package/src/search/postgres/values.ts +33 -0
- package/src/search/postgres/visibility.ts +54 -0
- package/src/search/requestFilters.ts +178 -0
- package/src/search/visibility.ts +193 -0
- package/src/storage/backend.ts +75 -0
- package/src/storage/deliver.ts +65 -0
- package/src/storage/directory/directory.ts +39 -0
- package/src/storage/entity-meta/entity-meta.ts +25 -0
- package/src/storage/location.ts +78 -0
- package/src/storage/ocfl/ocfl.ts +71 -0
- package/src/storage/results.ts +32 -0
- package/src/storage/s3/s3.ts +79 -0
- package/src/transformers/default.ts +115 -85
- package/src/types/access.ts +25 -0
- package/src/types/context.ts +15 -0
- package/src/types/transformers.ts +8 -34
- package/src/types/visibility.ts +21 -0
- package/src/utils/access.ts +80 -0
- package/src/utils/errors.ts +68 -11
- package/src/utils/headers.ts +22 -2
- package/src/utils/validation.ts +11 -0
- package/src/utils/visibility.ts +56 -0
- package/dist/app.test.d.ts +0 -1
- package/dist/app.test.js +0 -89
- package/dist/app.test.js.map +0 -1
- package/dist/express.d.ts +0 -4
- package/dist/express.js +0 -14
- package/dist/express.js.map +0 -1
- package/dist/index.dev.d.ts +0 -1
- package/dist/index.dev.js +0 -156
- package/dist/index.dev.js.map +0 -1
- package/dist/routes/crate.d.ts +0 -9
- package/dist/routes/crate.js +0 -82
- package/dist/routes/crate.js.map +0 -1
- package/dist/routes/crate.test.d.ts +0 -1
- package/dist/routes/crate.test.js +0 -342
- package/dist/routes/crate.test.js.map +0 -1
- package/dist/routes/entities.test.d.ts +0 -1
- package/dist/routes/entities.test.js +0 -273
- package/dist/routes/entities.test.js.map +0 -1
- package/dist/routes/entity.test.d.ts +0 -1
- package/dist/routes/entity.test.js +0 -130
- package/dist/routes/entity.test.js.map +0 -1
- package/dist/routes/file.test.d.ts +0 -1
- package/dist/routes/file.test.js +0 -315
- package/dist/routes/file.test.js.map +0 -1
- package/dist/routes/files.test.d.ts +0 -1
- package/dist/routes/files.test.js +0 -222
- package/dist/routes/files.test.js.map +0 -1
- package/dist/routes/search.test.d.ts +0 -1
- package/dist/routes/search.test.js +0 -780
- package/dist/routes/search.test.js.map +0 -1
- package/dist/test/helpers/fastify.d.ts +0 -9
- package/dist/test/helpers/fastify.js +0 -18
- package/dist/test/helpers/fastify.js.map +0 -1
- package/dist/test/integration.setup.d.ts +0 -29
- package/dist/test/integration.setup.js +0 -188
- package/dist/test/integration.setup.js.map +0 -1
- package/dist/test/integration.test.d.ts +0 -1
- package/dist/test/integration.test.js +0 -396
- package/dist/test/integration.test.js.map +0 -1
- package/dist/transformers/default.test.d.ts +0 -1
- package/dist/transformers/default.test.js +0 -163
- package/dist/transformers/default.test.js.map +0 -1
- package/dist/transformers/transformer.test.d.ts +0 -1
- package/dist/transformers/transformer.test.js +0 -187
- package/dist/transformers/transformer.test.js.map +0 -1
- package/dist/types/fileHandlers.d.ts +0 -41
- package/dist/types/fileHandlers.js +0 -2
- package/dist/types/fileHandlers.js.map +0 -1
- package/dist/utils/errors.test.d.ts +0 -1
- package/dist/utils/errors.test.js +0 -66
- package/dist/utils/errors.test.js.map +0 -1
- package/dist/utils/queryBuilder.d.ts +0 -35
- package/dist/utils/queryBuilder.js +0 -98
- package/dist/utils/queryBuilder.js.map +0 -1
- package/prisma.config.ts +0 -9
- package/src/app.test.ts +0 -119
- package/src/express.ts +0 -21
- package/src/index.dev.ts +0 -200
- package/src/reset.d.ts +0 -1
- package/src/routes/__snapshots__/entities.test.ts.snap +0 -96
- package/src/routes/__snapshots__/entity.test.ts.snap +0 -74
- package/src/routes/__snapshots__/search.test.ts.snap +0 -165
- package/src/routes/crate.test.ts +0 -423
- package/src/routes/crate.ts +0 -123
- package/src/routes/entities.test.ts +0 -320
- package/src/routes/entity.test.ts +0 -160
- package/src/routes/file.test.ts +0 -395
- package/src/routes/files.test.ts +0 -278
- package/src/routes/search.test.ts +0 -903
- package/src/test/__snapshots__/integration.test.ts.snap +0 -18
- package/src/test/helpers/fastify.ts +0 -27
- package/src/test/integration.setup.ts +0 -210
- package/src/test/integration.test.ts +0 -482
- package/src/transformers/default.test.ts +0 -184
- package/src/transformers/transformer.test.ts +0 -252
- package/src/types/fileHandlers.ts +0 -65
- package/src/utils/errors.test.ts +0 -75
- package/src/utils/queryBuilder.ts +0 -132
package/src/routes/search.ts
CHANGED
|
@@ -1,57 +1,86 @@
|
|
|
1
|
-
import type { Client } from '@opensearch-project/opensearch';
|
|
2
|
-
import type { MultiBucketAggregateBaseFiltersBucket } from '@opensearch-project/opensearch/api/_types/_common.aggregations.js';
|
|
3
|
-
import type { Search_Request } from '@opensearch-project/opensearch/api/index.js';
|
|
4
1
|
import type { FastifyPluginAsync } from 'fastify';
|
|
5
2
|
import type { ZodTypeProvider } from 'fastify-type-provider-zod';
|
|
6
|
-
import { z } from 'zod
|
|
3
|
+
import { z } from 'zod';
|
|
7
4
|
import type { PrismaClient } from '../generated/prisma/client.js';
|
|
8
|
-
import {
|
|
9
|
-
import
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
}
|
|
5
|
+
import { normaliseQuery } from '../search/advanced.js';
|
|
6
|
+
import {
|
|
7
|
+
type SearchBackend,
|
|
8
|
+
type SearchExtra,
|
|
9
|
+
SearchQueryError,
|
|
10
|
+
type SearchRequest,
|
|
11
|
+
type SearchResult,
|
|
12
|
+
} from '../search/backend.js';
|
|
13
|
+
import {
|
|
14
|
+
boundingBoxSchema,
|
|
15
|
+
filterValueSchema,
|
|
16
|
+
geoViolations,
|
|
17
|
+
splitFilters,
|
|
18
|
+
type Violation,
|
|
19
|
+
} from '../search/requestFilters.js';
|
|
20
|
+
import { applyTransformers, authoriseEntities } from '../transformers/default.js';
|
|
21
|
+
import type { AccessFunction, AccessInfo } from '../types/access.js';
|
|
22
|
+
import type { EntityTransformer } from '../types/transformers.js';
|
|
23
|
+
import type { EntityVisibilityFilter, SearchVisibilityFilter } from '../types/visibility.js';
|
|
24
|
+
import { createError, createValidationError, ERROR_CODES } from '../utils/errors.js';
|
|
23
25
|
|
|
24
26
|
const searchParamsSchema = z.object({
|
|
25
27
|
searchType: z.enum(['basic', 'advanced']).default('basic'),
|
|
26
28
|
query: z.string(),
|
|
27
|
-
filters: z.record(z.string(),
|
|
29
|
+
filters: z.record(z.string(), filterValueSchema).default({}),
|
|
28
30
|
boundingBox: boundingBoxSchema.optional(),
|
|
29
31
|
geohashPrecision: z.number().int().min(1).max(12).optional(),
|
|
30
32
|
limit: z.number().int().min(1).max(1000).default(100),
|
|
31
33
|
offset: z.number().int().min(0).default(0),
|
|
32
|
-
sort: z.
|
|
34
|
+
sort: z.string().default('relevance'),
|
|
33
35
|
order: z.enum(['asc', 'desc']).default('asc'),
|
|
34
36
|
});
|
|
35
37
|
|
|
38
|
+
type SearchParams = z.infer<typeof searchParamsSchema>;
|
|
39
|
+
|
|
36
40
|
type SearchRouteOptions = {
|
|
37
41
|
prisma: PrismaClient;
|
|
38
|
-
|
|
39
|
-
|
|
42
|
+
search: SearchBackend;
|
|
43
|
+
access: AccessFunction;
|
|
40
44
|
entityTransformers?: EntityTransformer[];
|
|
41
|
-
|
|
42
|
-
|
|
45
|
+
entityVisibilityFilter: EntityVisibilityFilter;
|
|
46
|
+
searchVisibilityFilter: SearchVisibilityFilter;
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
// Highlights quote the Record's text, so a hit denied either part keeps only its score
|
|
50
|
+
const visibleExtra = (searchExtra: SearchExtra | undefined, { metadata, content }: AccessInfo) => {
|
|
51
|
+
if (!searchExtra || (metadata && content)) {
|
|
52
|
+
return searchExtra;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
const { highlight: _, ...rest } = searchExtra;
|
|
56
|
+
|
|
57
|
+
return Object.keys(rest).length > 0 ? rest : undefined;
|
|
43
58
|
};
|
|
44
59
|
|
|
45
60
|
const search: FastifyPluginAsync<SearchRouteOptions> = async (fastify, opts) => {
|
|
46
61
|
const {
|
|
47
62
|
prisma,
|
|
48
|
-
|
|
49
|
-
|
|
63
|
+
search: backend,
|
|
64
|
+
access,
|
|
50
65
|
entityTransformers = [],
|
|
51
|
-
|
|
52
|
-
|
|
66
|
+
entityVisibilityFilter,
|
|
67
|
+
searchVisibilityFilter,
|
|
53
68
|
} = opts;
|
|
54
|
-
const
|
|
69
|
+
const { capabilities } = backend;
|
|
70
|
+
const noFacets = Object.fromEntries(Object.keys(capabilities.facets).map((name) => [name, []]));
|
|
71
|
+
|
|
72
|
+
const violations = ({ searchType, sort }: SearchParams) => {
|
|
73
|
+
const found: Violation[] = [];
|
|
74
|
+
if (!Object.hasOwn(capabilities.searchTypes, searchType)) {
|
|
75
|
+
found.push({ field: '/searchType', message: `${searchType} search isn't supported`, value: searchType });
|
|
76
|
+
}
|
|
77
|
+
if (sort !== 'relevance' && !Object.hasOwn(capabilities.sorts, sort)) {
|
|
78
|
+
found.push({ field: '/sort', message: `${sort} isn't a declared sort`, value: sort });
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
return found;
|
|
82
|
+
};
|
|
83
|
+
|
|
55
84
|
fastify.withTypeProvider<ZodTypeProvider>().post(
|
|
56
85
|
'/search',
|
|
57
86
|
{
|
|
@@ -60,140 +89,91 @@ const search: FastifyPluginAsync<SearchRouteOptions> = async (fastify, opts) =>
|
|
|
60
89
|
},
|
|
61
90
|
},
|
|
62
91
|
async (request, reply) => {
|
|
63
|
-
const {
|
|
92
|
+
const { filters, violations: filterProblems } = splitFilters(capabilities.filters, request.body.filters);
|
|
93
|
+
const problems = [
|
|
94
|
+
...violations(request.body),
|
|
95
|
+
...filterProblems,
|
|
96
|
+
...geoViolations(capabilities.geo, request.body),
|
|
97
|
+
];
|
|
98
|
+
if (problems.length > 0) {
|
|
99
|
+
return reply.code(400).send(createValidationError('The request parameters are invalid', problems));
|
|
100
|
+
}
|
|
64
101
|
|
|
102
|
+
const { searchType, query, filters: _, boundingBox, geohashPrecision, ...paging } = request.body;
|
|
103
|
+
|
|
104
|
+
const searchQuery = normaliseQuery(searchType, query, capabilities.searchTypes.advanced?.fields ?? {});
|
|
105
|
+
if ('problem' in searchQuery) {
|
|
106
|
+
const { problem } = searchQuery;
|
|
107
|
+
|
|
108
|
+
return reply.code(422).send(createValidationError(problem, [{ field: 'query', message: problem }]));
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
const context = { request, fastify };
|
|
112
|
+
// Only the database query needs it, so the search needn't wait; awaited before any reply so its error wins
|
|
113
|
+
const visibilityWhere = Promise.resolve(entityVisibilityFilter(context));
|
|
114
|
+
visibilityWhere.catch(() => undefined);
|
|
115
|
+
const visibility = await searchVisibilityFilter(context);
|
|
116
|
+
|
|
117
|
+
const searchRequest: SearchRequest = {
|
|
118
|
+
...searchQuery,
|
|
119
|
+
filters,
|
|
120
|
+
...(boundingBox && { boundingBox }),
|
|
121
|
+
...(geohashPrecision !== undefined && { geohashPrecision }),
|
|
122
|
+
...paging,
|
|
123
|
+
visibility,
|
|
124
|
+
};
|
|
125
|
+
|
|
126
|
+
const start = performance.now();
|
|
127
|
+
let result: SearchResult;
|
|
65
128
|
try {
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
description: {},
|
|
75
|
-
},
|
|
76
|
-
},
|
|
77
|
-
sort: queryBuilder.buildSort(sort, order),
|
|
78
|
-
from: offset,
|
|
79
|
-
size: limit,
|
|
80
|
-
},
|
|
81
|
-
};
|
|
82
|
-
fastify.log.debug(opensearchQuery);
|
|
83
|
-
const response = await opensearch.search(opensearchQuery);
|
|
84
|
-
|
|
85
|
-
if (!response.body?.hits?.hits) {
|
|
86
|
-
throw new Error('Invalid search response: missing hits data');
|
|
129
|
+
result = await backend.search(searchRequest, context);
|
|
130
|
+
} catch (error) {
|
|
131
|
+
await visibilityWhere.catch((visibilityError: unknown) => {
|
|
132
|
+
fastify.log.error({ err: error }, 'Search backend failed');
|
|
133
|
+
throw visibilityError;
|
|
134
|
+
});
|
|
135
|
+
if (!(error instanceof SearchQueryError)) {
|
|
136
|
+
throw error;
|
|
87
137
|
}
|
|
88
138
|
|
|
89
|
-
|
|
139
|
+
fastify.log.warn(`Invalid search query: ${error.message}`);
|
|
90
140
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
in: entityIds,
|
|
95
|
-
},
|
|
96
|
-
},
|
|
97
|
-
include: { file: { select: { id: true } } },
|
|
98
|
-
});
|
|
141
|
+
return reply.code(422).send(createError(ERROR_CODES.VALIDATION_ERROR, error.message));
|
|
142
|
+
}
|
|
143
|
+
const searchTime = Math.round(performance.now() - start);
|
|
99
144
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
const entities = await Promise.all(
|
|
106
|
-
response.body.hits.hits.map(async (hit) => {
|
|
107
|
-
if (!hit._source?.id) {
|
|
108
|
-
throw new Error('Missing id in search hit');
|
|
109
|
-
}
|
|
110
|
-
|
|
111
|
-
const dbEntity = entityMap.get(hit._source.id);
|
|
112
|
-
if (!dbEntity) {
|
|
113
|
-
fastify.log.warn(`Entity ${hit._source.id} found in OpenSearch but not in database`);
|
|
114
|
-
|
|
115
|
-
return null;
|
|
116
|
-
}
|
|
117
|
-
|
|
118
|
-
const base = baseEntityTransformer(dbEntity);
|
|
119
|
-
const standardEntity = {
|
|
120
|
-
...base,
|
|
121
|
-
memberOf: base.memberOf ? (refMap.get(base.memberOf) ?? null) : null,
|
|
122
|
-
rootCollection: base.rootCollection ? (refMap.get(base.rootCollection) ?? null) : null,
|
|
123
|
-
};
|
|
124
|
-
const authorisedEntity = await accessTransformer(standardEntity, { request, fastify });
|
|
125
|
-
|
|
126
|
-
let result = authorisedEntity;
|
|
127
|
-
for (const transformer of entityTransformers) {
|
|
128
|
-
result = await transformer(result, { request, fastify });
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
// Add search-specific metadata
|
|
132
|
-
return {
|
|
133
|
-
...(result as Record<string, unknown>),
|
|
134
|
-
searchExtra: {
|
|
135
|
-
score: hit._score,
|
|
136
|
-
highlight: hit.highlight,
|
|
137
|
-
},
|
|
138
|
-
};
|
|
139
|
-
}),
|
|
140
|
-
).then((results) => results.filter(Boolean));
|
|
141
|
-
|
|
142
|
-
const facets: Record<string, Array<{ name: string; count: number }>> = {};
|
|
143
|
-
if (response.body.aggregations) {
|
|
144
|
-
Object.keys(response.body.aggregations).forEach((key) => {
|
|
145
|
-
if (key !== 'geohash_grid') {
|
|
146
|
-
const agg = response.body.aggregations?.[key] as MultiBucketAggregateBaseFiltersBucket;
|
|
147
|
-
if (agg?.buckets && Array.isArray(agg.buckets)) {
|
|
148
|
-
facets[key] = agg.buckets.map((bucket) => ({
|
|
149
|
-
name: bucket.key,
|
|
150
|
-
count: bucket.doc_count,
|
|
151
|
-
}));
|
|
152
|
-
}
|
|
153
|
-
}
|
|
154
|
-
});
|
|
155
|
-
}
|
|
145
|
+
const dbEntities = await prisma.entity.findMany({
|
|
146
|
+
where: {
|
|
147
|
+
AND: [{ id: { in: result.hits.map(({ id }) => id) } }, await visibilityWhere],
|
|
148
|
+
},
|
|
149
|
+
});
|
|
156
150
|
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
const geohashAgg = response.body.aggregations.geohash_grid as MultiBucketAggregateBaseFiltersBucket;
|
|
160
|
-
if (geohashAgg?.buckets && Array.isArray(geohashAgg.buckets)) {
|
|
161
|
-
geohashAgg.buckets.forEach((bucket) => {
|
|
162
|
-
geohashGrid ||= {};
|
|
163
|
-
geohashGrid[bucket.key] = bucket.doc_count;
|
|
164
|
-
});
|
|
165
|
-
}
|
|
166
|
-
}
|
|
151
|
+
const authorisedEntities = await authoriseEntities(dbEntities, prisma, access, context);
|
|
152
|
+
const entityMap = new Map(authorisedEntities.map((entity, i) => [entity.id, { entity, row: dbEntities[i] }]));
|
|
167
153
|
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
const result = {
|
|
175
|
-
total,
|
|
176
|
-
searchTime: response.body.took,
|
|
177
|
-
entities,
|
|
178
|
-
facets: Object.keys(facets).length > 0 ? facets : undefined,
|
|
179
|
-
geohashGrid,
|
|
180
|
-
};
|
|
181
|
-
|
|
182
|
-
return result;
|
|
183
|
-
} catch (error) {
|
|
184
|
-
const err = error as Error;
|
|
154
|
+
const entities = await Promise.all(
|
|
155
|
+
result.hits.map(async ({ id, searchExtra }) => {
|
|
156
|
+
const found = entityMap.get(id);
|
|
157
|
+
if (!found) {
|
|
158
|
+
fastify.log.warn(`Entity ${id} found by the search backend but missing or not visible in database`);
|
|
185
159
|
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
fastify.log.warn(`Invalid search query: ${err.message}`);
|
|
160
|
+
return null;
|
|
161
|
+
}
|
|
189
162
|
|
|
190
|
-
|
|
191
|
-
|
|
163
|
+
const transformed = await applyTransformers(found.entity, entityTransformers, context, found.row);
|
|
164
|
+
const extra = visibleExtra(searchExtra, found.entity.access);
|
|
192
165
|
|
|
193
|
-
|
|
166
|
+
return extra === undefined ? transformed : { ...(transformed as object), searchExtra: extra };
|
|
167
|
+
}),
|
|
168
|
+
).then((results) => results.filter((entity) => entity !== null));
|
|
194
169
|
|
|
195
|
-
|
|
196
|
-
|
|
170
|
+
return {
|
|
171
|
+
total: result.total,
|
|
172
|
+
searchTime,
|
|
173
|
+
entities,
|
|
174
|
+
facets: { ...noFacets, ...result.facets },
|
|
175
|
+
...(result.geohashGrid && { geohashGrid: result.geohashGrid }),
|
|
176
|
+
};
|
|
197
177
|
},
|
|
198
178
|
);
|
|
199
179
|
};
|
|
@@ -0,0 +1,258 @@
|
|
|
1
|
+
import { type AdvancedQuery, type SearchQuery, SearchQueryError } from './backend.js';
|
|
2
|
+
import { dateSpan } from './fields.js';
|
|
3
|
+
|
|
4
|
+
/** In code points, for basic and advanced queries */
|
|
5
|
+
const MAX_QUERY_LENGTH = 1024;
|
|
6
|
+
const MAX_DEPTH = 10;
|
|
7
|
+
/** Leaf clauses, counted after field groups are rewritten */
|
|
8
|
+
const MAX_CLAUSES = 64;
|
|
9
|
+
|
|
10
|
+
class InvalidQueryError extends Error {
|
|
11
|
+
override name = 'InvalidQueryError';
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
type Token = {
|
|
15
|
+
kind: '(' | ')' | ':' | 'phrase' | 'word' | 'AND' | 'OR' | 'NOT' | 'end';
|
|
16
|
+
text: string;
|
|
17
|
+
/** 1-based, in code points */
|
|
18
|
+
position: number;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
const OPERATORS: readonly string[] = ['AND', 'OR', 'NOT'];
|
|
22
|
+
const PUNCTUATION: readonly string[] = ['(', ')', ':'];
|
|
23
|
+
|
|
24
|
+
const isSpace = (char: string) => /\s/u.test(char);
|
|
25
|
+
const endsWord = (char: string) => isSpace(char) || PUNCTUATION.includes(char) || char === '"';
|
|
26
|
+
|
|
27
|
+
const tokenise = (chars: string[]) => {
|
|
28
|
+
const tokens: Token[] = [];
|
|
29
|
+
let at = 0;
|
|
30
|
+
|
|
31
|
+
while (at < chars.length) {
|
|
32
|
+
const char = chars[at];
|
|
33
|
+
const position = at + 1;
|
|
34
|
+
|
|
35
|
+
if (isSpace(char)) {
|
|
36
|
+
at++;
|
|
37
|
+
} else if (PUNCTUATION.includes(char)) {
|
|
38
|
+
tokens.push({ kind: char as Token['kind'], text: char, position });
|
|
39
|
+
at++;
|
|
40
|
+
} else if (char === '"') {
|
|
41
|
+
const close = chars.indexOf('"', at + 1);
|
|
42
|
+
if (close === -1) {
|
|
43
|
+
throw new InvalidQueryError(
|
|
44
|
+
`Expected a closing " for the phrase at position ${position}, found the end of the query`,
|
|
45
|
+
);
|
|
46
|
+
}
|
|
47
|
+
tokens.push({ kind: 'phrase', text: chars.slice(at + 1, close).join(''), position });
|
|
48
|
+
at = close + 1;
|
|
49
|
+
} else {
|
|
50
|
+
const start = at;
|
|
51
|
+
while (at < chars.length && !endsWord(chars[at])) {
|
|
52
|
+
at++;
|
|
53
|
+
}
|
|
54
|
+
const text = chars.slice(start, at).join('');
|
|
55
|
+
tokens.push({ kind: OPERATORS.includes(text) ? (text as Token['kind']) : 'word', text, position });
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
tokens.push({ kind: 'end', text: '', position: chars.length + 1 });
|
|
60
|
+
|
|
61
|
+
return tokens;
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
const describe = ({ kind, text }: Token) => {
|
|
65
|
+
switch (kind) {
|
|
66
|
+
case 'end':
|
|
67
|
+
return 'the end of the query';
|
|
68
|
+
case 'phrase':
|
|
69
|
+
return `the phrase "${text}"`;
|
|
70
|
+
case 'word':
|
|
71
|
+
return `"${text}"`;
|
|
72
|
+
default:
|
|
73
|
+
return kind.length === 1 ? `"${kind}"` : kind;
|
|
74
|
+
}
|
|
75
|
+
};
|
|
76
|
+
|
|
77
|
+
const combine = (type: 'and' | 'or', parts: AdvancedQuery[]): AdvancedQuery =>
|
|
78
|
+
parts.length === 1
|
|
79
|
+
? parts[0]
|
|
80
|
+
: { type, clauses: parts.flatMap((part) => (part.type === type ? part.clauses : [part])) };
|
|
81
|
+
|
|
82
|
+
const parse = (query: string, fields: Record<string, unknown>): AdvancedQuery => {
|
|
83
|
+
const tokens = tokenise([...query]);
|
|
84
|
+
let at = 0;
|
|
85
|
+
|
|
86
|
+
const peek = (offset = 0) => tokens[at + offset];
|
|
87
|
+
const fail = (expected: string): never => {
|
|
88
|
+
throw new InvalidQueryError(`Expected ${expected} at position ${peek().position}, found ${describe(peek())}`);
|
|
89
|
+
};
|
|
90
|
+
const expect = (kind: Token['kind'], expected: string) => {
|
|
91
|
+
if (peek().kind !== kind) {
|
|
92
|
+
fail(expected);
|
|
93
|
+
}
|
|
94
|
+
at++;
|
|
95
|
+
};
|
|
96
|
+
|
|
97
|
+
// `field` is set inside a field group, where clauses are values rather than field matches
|
|
98
|
+
const parseOr = (field?: string): AdvancedQuery => {
|
|
99
|
+
const parts = [parseAnd(field)];
|
|
100
|
+
while (peek().kind === 'OR') {
|
|
101
|
+
at++;
|
|
102
|
+
parts.push(parseAnd(field));
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
return combine('or', parts);
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
const parseAnd = (field?: string): AdvancedQuery => {
|
|
109
|
+
const parts = [parseUnary(field)];
|
|
110
|
+
while (peek().kind === 'AND') {
|
|
111
|
+
at++;
|
|
112
|
+
parts.push(parseUnary(field));
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
return combine('and', parts);
|
|
116
|
+
};
|
|
117
|
+
|
|
118
|
+
const parseUnary = (field?: string): AdvancedQuery => {
|
|
119
|
+
if (peek().kind === 'NOT') {
|
|
120
|
+
at++;
|
|
121
|
+
|
|
122
|
+
return { type: 'not', clause: parseUnary(field) };
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
return parsePrimary(field);
|
|
126
|
+
};
|
|
127
|
+
|
|
128
|
+
const parseBracketed = (field?: string) => {
|
|
129
|
+
at++;
|
|
130
|
+
const clause = parseOr(field);
|
|
131
|
+
expect(')', 'AND, OR or ")"');
|
|
132
|
+
|
|
133
|
+
return clause;
|
|
134
|
+
};
|
|
135
|
+
|
|
136
|
+
const parsePrimary = (field?: string): AdvancedQuery => {
|
|
137
|
+
if (peek().kind === '(') {
|
|
138
|
+
return parseBracketed(field);
|
|
139
|
+
}
|
|
140
|
+
if (field !== undefined) {
|
|
141
|
+
return parseValue(field);
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
const name = peek();
|
|
145
|
+
if (name.kind !== 'word') {
|
|
146
|
+
fail('a field name');
|
|
147
|
+
}
|
|
148
|
+
at++;
|
|
149
|
+
expect(':', `":" after the field name "${name.text}"`);
|
|
150
|
+
if (!Object.hasOwn(fields, name.text)) {
|
|
151
|
+
throw new InvalidQueryError(`"${name.text}" at position ${name.position} isn't a declared advanced field`);
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
return peek().kind === '(' ? parseBracketed(name.text) : parseValue(name.text);
|
|
155
|
+
};
|
|
156
|
+
|
|
157
|
+
const startsValueWord = () => peek().kind === 'word' && peek(1).kind !== ':';
|
|
158
|
+
|
|
159
|
+
const parseValue = (field: string): AdvancedQuery => {
|
|
160
|
+
if (peek().kind === 'phrase') {
|
|
161
|
+
const { text } = peek();
|
|
162
|
+
at++;
|
|
163
|
+
|
|
164
|
+
return { type: 'match', field, value: text, phrase: true };
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
if (!startsValueWord()) {
|
|
168
|
+
fail(`a value for "${field}"`);
|
|
169
|
+
}
|
|
170
|
+
const words: string[] = [];
|
|
171
|
+
while (startsValueWord()) {
|
|
172
|
+
words.push(peek().text);
|
|
173
|
+
at++;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
return { type: 'match', field, value: words.join(' '), phrase: false };
|
|
177
|
+
};
|
|
178
|
+
|
|
179
|
+
const tree = parseOr();
|
|
180
|
+
expect('end', 'AND, OR or the end of the query');
|
|
181
|
+
|
|
182
|
+
return tree;
|
|
183
|
+
};
|
|
184
|
+
|
|
185
|
+
const depthOf = (query: AdvancedQuery): number => {
|
|
186
|
+
switch (query.type) {
|
|
187
|
+
case 'and':
|
|
188
|
+
case 'or':
|
|
189
|
+
return 1 + Math.max(...query.clauses.map(depthOf));
|
|
190
|
+
case 'not':
|
|
191
|
+
return 1 + depthOf(query.clause);
|
|
192
|
+
default:
|
|
193
|
+
return 0;
|
|
194
|
+
}
|
|
195
|
+
};
|
|
196
|
+
|
|
197
|
+
const clausesIn = (query: AdvancedQuery): number => {
|
|
198
|
+
switch (query.type) {
|
|
199
|
+
case 'and':
|
|
200
|
+
case 'or':
|
|
201
|
+
return query.clauses.reduce((sum, clause) => sum + clausesIn(clause), 0);
|
|
202
|
+
case 'not':
|
|
203
|
+
return clausesIn(query.clause);
|
|
204
|
+
default:
|
|
205
|
+
return 1;
|
|
206
|
+
}
|
|
207
|
+
};
|
|
208
|
+
|
|
209
|
+
const parseWithinCaps = (query: string, advancedFields: Record<string, unknown>): AdvancedQuery => {
|
|
210
|
+
const tree = parse(query, advancedFields);
|
|
211
|
+
if (depthOf(tree) > MAX_DEPTH) {
|
|
212
|
+
throw new InvalidQueryError(`The query nests deeper than the limit of ${MAX_DEPTH} levels`);
|
|
213
|
+
}
|
|
214
|
+
if (clausesIn(tree) > MAX_CLAUSES) {
|
|
215
|
+
throw new InvalidQueryError(`The query has more than the limit of ${MAX_CLAUSES} clauses`);
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
return tree;
|
|
219
|
+
};
|
|
220
|
+
|
|
221
|
+
/**
|
|
222
|
+
* Check a query against the caps and parse an advanced one into a tree over `advancedFields`
|
|
223
|
+
* A query arocapi refuses gives the problem instead
|
|
224
|
+
*/
|
|
225
|
+
export const normaliseQuery = (
|
|
226
|
+
searchType: 'basic' | 'advanced',
|
|
227
|
+
query: string,
|
|
228
|
+
advancedFields: Record<string, unknown>,
|
|
229
|
+
): SearchQuery | { problem: string } => {
|
|
230
|
+
if ([...query].length > MAX_QUERY_LENGTH) {
|
|
231
|
+
return { problem: `The query is longer than the limit of ${MAX_QUERY_LENGTH} characters` };
|
|
232
|
+
}
|
|
233
|
+
if (searchType === 'basic') {
|
|
234
|
+
return { searchType, query };
|
|
235
|
+
}
|
|
236
|
+
if (query.trim() === '') {
|
|
237
|
+
return { searchType, query: { type: 'all' } };
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
try {
|
|
241
|
+
return { searchType, query: parseWithinCaps(query, advancedFields) };
|
|
242
|
+
} catch (error) {
|
|
243
|
+
// Only InvalidQueryError is thrown: the parser is pure and its recursion is bounded by the length cap
|
|
244
|
+
return { problem: (error as InvalidQueryError).message };
|
|
245
|
+
}
|
|
246
|
+
};
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* The UTC instants an advanced match on a date field covers, throwing a SearchQueryError for a value that isn't a date
|
|
250
|
+
*/
|
|
251
|
+
export const matchedDateSpan = (field: string, value: string) => {
|
|
252
|
+
const span = dateSpan(value);
|
|
253
|
+
if (!span) {
|
|
254
|
+
throw new SearchQueryError(`"${field}" is a date field: "${value}" isn't a date`);
|
|
255
|
+
}
|
|
256
|
+
|
|
257
|
+
return span;
|
|
258
|
+
};
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
import type { RequestContext } from '../types/context.js';
|
|
2
|
+
import type { GeoPoint, SearchDocument } from './fields.js';
|
|
3
|
+
import type { SearchVisibilityWhere } from './visibility.js';
|
|
4
|
+
|
|
5
|
+
type Labelled = { label?: string };
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* What a search backend publishes in `/capabilities` and arocapi validates requests against
|
|
9
|
+
*/
|
|
10
|
+
export type BackendSearchCapability = {
|
|
11
|
+
filters: Record<string, Labelled & { type: 'string' | 'date' | 'number' | 'boolean' }>;
|
|
12
|
+
facets: Record<string, Labelled>;
|
|
13
|
+
sorts: Record<string, Labelled>;
|
|
14
|
+
searchTypes: { basic: Record<string, never>; advanced?: { fields: Record<string, Labelled> } };
|
|
15
|
+
geo: { supported: boolean };
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* A parsed advanced query
|
|
20
|
+
*/
|
|
21
|
+
export type AdvancedQuery =
|
|
22
|
+
| { type: 'and' | 'or'; clauses: AdvancedQuery[] }
|
|
23
|
+
| { type: 'not'; clause: AdvancedQuery }
|
|
24
|
+
| { type: 'match'; field: string; value: string; phrase: boolean }
|
|
25
|
+
| { type: 'all' };
|
|
26
|
+
|
|
27
|
+
/** Date bounds are UTC instants */
|
|
28
|
+
export type RangeBound = number | Date;
|
|
29
|
+
|
|
30
|
+
export type RangeFilter = { gte?: RangeBound; lte?: RangeBound };
|
|
31
|
+
|
|
32
|
+
export type BoundingBox = { topRight: GeoPoint; bottomLeft: GeoPoint };
|
|
33
|
+
|
|
34
|
+
export type SearchQuery = { searchType: 'basic'; query: string } | { searchType: 'advanced'; query: AdvancedQuery };
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* A validated, normalised search, as arocapi hands it to a search backend
|
|
38
|
+
*/
|
|
39
|
+
export type SearchRequest = SearchQuery & {
|
|
40
|
+
filters: {
|
|
41
|
+
/** Field name → values, any of which matches */
|
|
42
|
+
terms: Record<string, string[]>;
|
|
43
|
+
/** Field name → ranges, any of which matches; date and number filters always arrive here, exact values as ranges */
|
|
44
|
+
ranges: Record<string, RangeFilter[]>;
|
|
45
|
+
};
|
|
46
|
+
boundingBox?: BoundingBox;
|
|
47
|
+
geohashPrecision?: number;
|
|
48
|
+
limit: number;
|
|
49
|
+
offset: number;
|
|
50
|
+
/** A declared sort, or `relevance`: best match first, stable for an empty query */
|
|
51
|
+
sort: string;
|
|
52
|
+
order: 'asc' | 'desc';
|
|
53
|
+
/** The resolved `searchVisibilityFilter`, which the backend checks and applies to hits, total and facets */
|
|
54
|
+
visibility: SearchVisibilityWhere;
|
|
55
|
+
};
|
|
56
|
+
|
|
57
|
+
export type SearchExtra = { score?: number; highlight?: Record<string, string[]> };
|
|
58
|
+
|
|
59
|
+
export type FacetBucket = { name: string; count: number };
|
|
60
|
+
|
|
61
|
+
/** The most geohash cells a reference backend returns: those with the most hits */
|
|
62
|
+
export const GEOHASH_GRID_CELLS = 10_000;
|
|
63
|
+
|
|
64
|
+
export type SearchResult = {
|
|
65
|
+
/** Exact */
|
|
66
|
+
total: number;
|
|
67
|
+
/** Ordered */
|
|
68
|
+
hits: { id: string; searchExtra?: SearchExtra }[];
|
|
69
|
+
facets?: Record<string, FacetBucket[]>;
|
|
70
|
+
geohashGrid?: Record<string, number>;
|
|
71
|
+
};
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* A search backend: arocapi validates each request against `capabilities`, the backend only searches
|
|
75
|
+
*/
|
|
76
|
+
export interface SearchBackend {
|
|
77
|
+
/** What `/capabilities` publishes and arocapi validates requests against */
|
|
78
|
+
readonly capabilities: BackendSearchCapability;
|
|
79
|
+
search(request: SearchRequest, context: RequestContext): Promise<SearchResult>;
|
|
80
|
+
/** Upsert search documents; each entity must exist first */
|
|
81
|
+
index(docs: SearchDocument[], options?: { tx?: unknown }): Promise<void>;
|
|
82
|
+
/** Remove search documents, ignoring ids that aren't indexed */
|
|
83
|
+
remove(ids: string[]): Promise<void>;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/**
|
|
87
|
+
* Thrown by a search backend for a query it can't run; arocapi answers 422 with its message
|
|
88
|
+
*/
|
|
89
|
+
export class SearchQueryError extends Error {
|
|
90
|
+
override name = 'SearchQueryError';
|
|
91
|
+
}
|