arocapi 4.0.1 → 5.0.0-beta.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (242) hide show
  1. package/LICENSE.md +1 -1
  2. package/README.md +37 -781
  3. package/dist/app.d.ts +38 -21
  4. package/dist/app.js +63 -45
  5. package/dist/app.js.map +1 -1
  6. package/dist/routes/capabilities.d.ts +7 -0
  7. package/dist/routes/capabilities.js +13 -0
  8. package/dist/routes/capabilities.js.map +1 -0
  9. package/dist/routes/entities.d.ts +5 -2
  10. package/dist/routes/entities.js +38 -58
  11. package/dist/routes/entities.js.map +1 -1
  12. package/dist/routes/entity.d.ts +7 -2
  13. package/dist/routes/entity.js +14 -30
  14. package/dist/routes/entity.js.map +1 -1
  15. package/dist/routes/file.d.ts +8 -2
  16. package/dist/routes/file.js +38 -73
  17. package/dist/routes/file.js.map +1 -1
  18. package/dist/routes/files.d.ts +5 -2
  19. package/dist/routes/files.js +25 -40
  20. package/dist/routes/files.js.map +1 -1
  21. package/dist/routes/metadata.d.ts +15 -0
  22. package/dist/routes/metadata.js +38 -0
  23. package/dist/routes/metadata.js.map +1 -0
  24. package/dist/routes/search.d.ts +8 -7
  25. package/dist/routes/search.js +91 -120
  26. package/dist/routes/search.js.map +1 -1
  27. package/dist/search/advanced.d.ts +8 -0
  28. package/dist/search/advanced.js +198 -0
  29. package/dist/search/advanced.js.map +1 -0
  30. package/dist/search/backend.d.ts +95 -0
  31. package/dist/search/backend.js +5 -0
  32. package/dist/search/backend.js.map +1 -0
  33. package/dist/search/fields.d.ts +63 -0
  34. package/dist/search/fields.js +203 -0
  35. package/dist/search/fields.js.map +1 -0
  36. package/dist/search/opensearch/advanced.d.ts +4 -0
  37. package/dist/search/opensearch/advanced.js +28 -0
  38. package/dist/search/opensearch/advanced.js.map +1 -0
  39. package/dist/search/opensearch/opensearch.d.ts +16 -0
  40. package/dist/search/opensearch/opensearch.js +187 -0
  41. package/dist/search/opensearch/opensearch.js.map +1 -0
  42. package/dist/search/opensearch/visibility.d.ts +3 -0
  43. package/dist/search/opensearch/visibility.js +27 -0
  44. package/dist/search/opensearch/visibility.js.map +1 -0
  45. package/dist/search/postgres/advanced.d.ts +7 -0
  46. package/dist/search/postgres/advanced.js +54 -0
  47. package/dist/search/postgres/advanced.js.map +1 -0
  48. package/dist/search/postgres/facets.d.ts +2 -0
  49. package/dist/search/postgres/facets.js +27 -0
  50. package/dist/search/postgres/facets.js.map +1 -0
  51. package/dist/search/postgres/geo.d.ts +6 -0
  52. package/dist/search/postgres/geo.js +42 -0
  53. package/dist/search/postgres/geo.js.map +1 -0
  54. package/dist/search/postgres/postgres.d.ts +19 -0
  55. package/dist/search/postgres/postgres.js +157 -0
  56. package/dist/search/postgres/postgres.js.map +1 -0
  57. package/dist/search/postgres/values.d.ts +8 -0
  58. package/dist/search/postgres/values.js +20 -0
  59. package/dist/search/postgres/values.js.map +1 -0
  60. package/dist/search/postgres/visibility.d.ts +4 -0
  61. package/dist/search/postgres/visibility.js +38 -0
  62. package/dist/search/postgres/visibility.js.map +1 -0
  63. package/dist/search/requestFilters.d.ts +39 -0
  64. package/dist/search/requestFilters.js +125 -0
  65. package/dist/search/requestFilters.js.map +1 -0
  66. package/dist/search/visibility.d.ts +51 -0
  67. package/dist/search/visibility.js +100 -0
  68. package/dist/search/visibility.js.map +1 -0
  69. package/dist/storage/backend.d.ts +58 -0
  70. package/dist/storage/backend.js +2 -0
  71. package/dist/storage/backend.js.map +1 -0
  72. package/dist/storage/deliver.d.ts +12 -0
  73. package/dist/storage/deliver.js +34 -0
  74. package/dist/storage/deliver.js.map +1 -0
  75. package/dist/storage/directory/directory.d.ts +6 -0
  76. package/dist/storage/directory/directory.js +24 -0
  77. package/dist/storage/directory/directory.js.map +1 -0
  78. package/dist/storage/entity-meta/entity-meta.d.ts +5 -0
  79. package/dist/storage/entity-meta/entity-meta.js +14 -0
  80. package/dist/storage/entity-meta/entity-meta.js.map +1 -0
  81. package/dist/storage/location.d.ts +19 -0
  82. package/dist/storage/location.js +36 -0
  83. package/dist/storage/location.js.map +1 -0
  84. package/dist/storage/ocfl/ocfl.d.ts +6 -0
  85. package/dist/storage/ocfl/ocfl.js +45 -0
  86. package/dist/storage/ocfl/ocfl.js.map +1 -0
  87. package/dist/storage/results.d.ts +7 -0
  88. package/dist/storage/results.js +22 -0
  89. package/dist/storage/results.js.map +1 -0
  90. package/dist/storage/s3/s3.d.ts +10 -0
  91. package/dist/storage/s3/s3.js +47 -0
  92. package/dist/storage/s3/s3.js.map +1 -0
  93. package/dist/transformers/default.d.ts +11 -22
  94. package/dist/transformers/default.js +41 -15
  95. package/dist/transformers/default.js.map +1 -1
  96. package/dist/types/access.d.ts +9 -0
  97. package/dist/types/access.js +2 -0
  98. package/dist/types/access.js.map +1 -0
  99. package/dist/types/context.d.ts +6 -0
  100. package/dist/types/context.js +2 -0
  101. package/dist/types/context.js.map +1 -0
  102. package/dist/types/transformers.d.ts +4 -10
  103. package/dist/types/visibility.d.ts +6 -0
  104. package/dist/types/visibility.js +2 -0
  105. package/dist/types/visibility.js.map +1 -0
  106. package/dist/utils/access.d.ts +17 -0
  107. package/dist/utils/access.js +41 -0
  108. package/dist/utils/access.js.map +1 -0
  109. package/dist/utils/errors.d.ts +19 -10
  110. package/dist/utils/errors.js +44 -8
  111. package/dist/utils/errors.js.map +1 -1
  112. package/dist/utils/headers.d.ts +1 -1
  113. package/dist/utils/headers.js +10 -1
  114. package/dist/utils/headers.js.map +1 -1
  115. package/dist/utils/validation.d.ts +2 -0
  116. package/dist/utils/validation.js +8 -0
  117. package/dist/utils/validation.js.map +1 -0
  118. package/dist/utils/visibility.d.ts +18 -0
  119. package/dist/utils/visibility.js +16 -0
  120. package/dist/utils/visibility.js.map +1 -0
  121. package/package.json +84 -45
  122. package/prisma/models/entity.prisma +9 -8
  123. package/prisma/models/entitySearch.prisma +21 -0
  124. package/prisma/models/file.prisma +3 -3
  125. package/src/app.ts +169 -102
  126. package/src/routes/capabilities.ts +22 -0
  127. package/src/routes/entities.ts +49 -68
  128. package/src/routes/entity.ts +22 -36
  129. package/src/routes/file.ts +54 -104
  130. package/src/routes/files.ts +32 -48
  131. package/src/routes/metadata.ts +67 -0
  132. package/src/routes/search.ts +132 -152
  133. package/src/search/advanced.ts +258 -0
  134. package/src/search/backend.ts +91 -0
  135. package/src/search/fields.ts +320 -0
  136. package/src/search/opensearch/advanced.ts +36 -0
  137. package/src/search/opensearch/opensearch.ts +290 -0
  138. package/src/search/opensearch/visibility.ts +38 -0
  139. package/src/search/postgres/advanced.ts +76 -0
  140. package/src/search/postgres/facets.ts +34 -0
  141. package/src/search/postgres/geo.ts +58 -0
  142. package/src/search/postgres/postgres.ts +232 -0
  143. package/src/search/postgres/values.ts +33 -0
  144. package/src/search/postgres/visibility.ts +54 -0
  145. package/src/search/requestFilters.ts +178 -0
  146. package/src/search/visibility.ts +193 -0
  147. package/src/storage/backend.ts +75 -0
  148. package/src/storage/deliver.ts +65 -0
  149. package/src/storage/directory/directory.ts +39 -0
  150. package/src/storage/entity-meta/entity-meta.ts +25 -0
  151. package/src/storage/location.ts +78 -0
  152. package/src/storage/ocfl/ocfl.ts +71 -0
  153. package/src/storage/results.ts +32 -0
  154. package/src/storage/s3/s3.ts +79 -0
  155. package/src/transformers/default.ts +115 -85
  156. package/src/types/access.ts +25 -0
  157. package/src/types/context.ts +15 -0
  158. package/src/types/transformers.ts +8 -34
  159. package/src/types/visibility.ts +21 -0
  160. package/src/utils/access.ts +80 -0
  161. package/src/utils/errors.ts +68 -11
  162. package/src/utils/headers.ts +22 -2
  163. package/src/utils/validation.ts +11 -0
  164. package/src/utils/visibility.ts +56 -0
  165. package/dist/app.test.d.ts +0 -1
  166. package/dist/app.test.js +0 -89
  167. package/dist/app.test.js.map +0 -1
  168. package/dist/express.d.ts +0 -4
  169. package/dist/express.js +0 -14
  170. package/dist/express.js.map +0 -1
  171. package/dist/index.dev.d.ts +0 -1
  172. package/dist/index.dev.js +0 -156
  173. package/dist/index.dev.js.map +0 -1
  174. package/dist/routes/crate.d.ts +0 -9
  175. package/dist/routes/crate.js +0 -82
  176. package/dist/routes/crate.js.map +0 -1
  177. package/dist/routes/crate.test.d.ts +0 -1
  178. package/dist/routes/crate.test.js +0 -342
  179. package/dist/routes/crate.test.js.map +0 -1
  180. package/dist/routes/entities.test.d.ts +0 -1
  181. package/dist/routes/entities.test.js +0 -273
  182. package/dist/routes/entities.test.js.map +0 -1
  183. package/dist/routes/entity.test.d.ts +0 -1
  184. package/dist/routes/entity.test.js +0 -130
  185. package/dist/routes/entity.test.js.map +0 -1
  186. package/dist/routes/file.test.d.ts +0 -1
  187. package/dist/routes/file.test.js +0 -315
  188. package/dist/routes/file.test.js.map +0 -1
  189. package/dist/routes/files.test.d.ts +0 -1
  190. package/dist/routes/files.test.js +0 -222
  191. package/dist/routes/files.test.js.map +0 -1
  192. package/dist/routes/search.test.d.ts +0 -1
  193. package/dist/routes/search.test.js +0 -780
  194. package/dist/routes/search.test.js.map +0 -1
  195. package/dist/test/helpers/fastify.d.ts +0 -9
  196. package/dist/test/helpers/fastify.js +0 -18
  197. package/dist/test/helpers/fastify.js.map +0 -1
  198. package/dist/test/integration.setup.d.ts +0 -29
  199. package/dist/test/integration.setup.js +0 -188
  200. package/dist/test/integration.setup.js.map +0 -1
  201. package/dist/test/integration.test.d.ts +0 -1
  202. package/dist/test/integration.test.js +0 -396
  203. package/dist/test/integration.test.js.map +0 -1
  204. package/dist/transformers/default.test.d.ts +0 -1
  205. package/dist/transformers/default.test.js +0 -163
  206. package/dist/transformers/default.test.js.map +0 -1
  207. package/dist/transformers/transformer.test.d.ts +0 -1
  208. package/dist/transformers/transformer.test.js +0 -187
  209. package/dist/transformers/transformer.test.js.map +0 -1
  210. package/dist/types/fileHandlers.d.ts +0 -41
  211. package/dist/types/fileHandlers.js +0 -2
  212. package/dist/types/fileHandlers.js.map +0 -1
  213. package/dist/utils/errors.test.d.ts +0 -1
  214. package/dist/utils/errors.test.js +0 -66
  215. package/dist/utils/errors.test.js.map +0 -1
  216. package/dist/utils/queryBuilder.d.ts +0 -35
  217. package/dist/utils/queryBuilder.js +0 -98
  218. package/dist/utils/queryBuilder.js.map +0 -1
  219. package/prisma.config.ts +0 -9
  220. package/src/app.test.ts +0 -119
  221. package/src/express.ts +0 -21
  222. package/src/index.dev.ts +0 -200
  223. package/src/reset.d.ts +0 -1
  224. package/src/routes/__snapshots__/entities.test.ts.snap +0 -96
  225. package/src/routes/__snapshots__/entity.test.ts.snap +0 -74
  226. package/src/routes/__snapshots__/search.test.ts.snap +0 -165
  227. package/src/routes/crate.test.ts +0 -423
  228. package/src/routes/crate.ts +0 -123
  229. package/src/routes/entities.test.ts +0 -320
  230. package/src/routes/entity.test.ts +0 -160
  231. package/src/routes/file.test.ts +0 -395
  232. package/src/routes/files.test.ts +0 -278
  233. package/src/routes/search.test.ts +0 -903
  234. package/src/test/__snapshots__/integration.test.ts.snap +0 -18
  235. package/src/test/helpers/fastify.ts +0 -27
  236. package/src/test/integration.setup.ts +0 -210
  237. package/src/test/integration.test.ts +0 -482
  238. package/src/transformers/default.test.ts +0 -184
  239. package/src/transformers/transformer.test.ts +0 -252
  240. package/src/types/fileHandlers.ts +0 -65
  241. package/src/utils/errors.test.ts +0 -75
  242. package/src/utils/queryBuilder.ts +0 -132
@@ -1,57 +1,86 @@
1
- import type { Client } from '@opensearch-project/opensearch';
2
- import type { MultiBucketAggregateBaseFiltersBucket } from '@opensearch-project/opensearch/api/_types/_common.aggregations.js';
3
- import type { Search_Request } from '@opensearch-project/opensearch/api/index.js';
4
1
  import type { FastifyPluginAsync } from 'fastify';
5
2
  import type { ZodTypeProvider } from 'fastify-type-provider-zod';
6
- import { z } from 'zod/v4';
3
+ import { z } from 'zod';
7
4
  import type { PrismaClient } from '../generated/prisma/client.js';
8
- import { baseEntityTransformer, resolveEntityReferences } from '../transformers/default.js';
9
- import type { AccessTransformer, EntityTransformer } from '../types/transformers.js';
10
- import { createInternalError, createInvalidRequestError } from '../utils/errors.js';
11
- import { OpensearchQueryBuilder, type QueryBuilderOptions } from '../utils/queryBuilder.js';
12
-
13
- const boundingBoxSchema = z.object({
14
- topRight: z.object({
15
- lat: z.number(),
16
- lng: z.number(),
17
- }),
18
- bottomLeft: z.object({
19
- lat: z.number(),
20
- lng: z.number(),
21
- }),
22
- });
5
+ import { normaliseQuery } from '../search/advanced.js';
6
+ import {
7
+ type SearchBackend,
8
+ type SearchExtra,
9
+ SearchQueryError,
10
+ type SearchRequest,
11
+ type SearchResult,
12
+ } from '../search/backend.js';
13
+ import {
14
+ boundingBoxSchema,
15
+ filterValueSchema,
16
+ geoViolations,
17
+ splitFilters,
18
+ type Violation,
19
+ } from '../search/requestFilters.js';
20
+ import { applyTransformers, authoriseEntities } from '../transformers/default.js';
21
+ import type { AccessFunction, AccessInfo } from '../types/access.js';
22
+ import type { EntityTransformer } from '../types/transformers.js';
23
+ import type { EntityVisibilityFilter, SearchVisibilityFilter } from '../types/visibility.js';
24
+ import { createError, createValidationError, ERROR_CODES } from '../utils/errors.js';
23
25
 
24
26
  const searchParamsSchema = z.object({
25
27
  searchType: z.enum(['basic', 'advanced']).default('basic'),
26
28
  query: z.string(),
27
- filters: z.record(z.string(), z.array(z.string())).optional(),
29
+ filters: z.record(z.string(), filterValueSchema).default({}),
28
30
  boundingBox: boundingBoxSchema.optional(),
29
31
  geohashPrecision: z.number().int().min(1).max(12).optional(),
30
32
  limit: z.number().int().min(1).max(1000).default(100),
31
33
  offset: z.number().int().min(0).default(0),
32
- sort: z.enum(['id', 'name', 'createdAt', 'updatedAt', 'relevance']).default('relevance'),
34
+ sort: z.string().default('relevance'),
33
35
  order: z.enum(['asc', 'desc']).default('asc'),
34
36
  });
35
37
 
38
+ type SearchParams = z.infer<typeof searchParamsSchema>;
39
+
36
40
  type SearchRouteOptions = {
37
41
  prisma: PrismaClient;
38
- opensearch: Client;
39
- accessTransformer: AccessTransformer;
42
+ search: SearchBackend;
43
+ access: AccessFunction;
40
44
  entityTransformers?: EntityTransformer[];
41
- queryBuilderClass?: typeof OpensearchQueryBuilder;
42
- queryBuilderOptions?: QueryBuilderOptions;
45
+ entityVisibilityFilter: EntityVisibilityFilter;
46
+ searchVisibilityFilter: SearchVisibilityFilter;
47
+ };
48
+
49
+ // Highlights quote the Record's text, so a hit denied either part keeps only its score
50
+ const visibleExtra = (searchExtra: SearchExtra | undefined, { metadata, content }: AccessInfo) => {
51
+ if (!searchExtra || (metadata && content)) {
52
+ return searchExtra;
53
+ }
54
+
55
+ const { highlight: _, ...rest } = searchExtra;
56
+
57
+ return Object.keys(rest).length > 0 ? rest : undefined;
43
58
  };
44
59
 
45
60
  const search: FastifyPluginAsync<SearchRouteOptions> = async (fastify, opts) => {
46
61
  const {
47
62
  prisma,
48
- opensearch,
49
- accessTransformer,
63
+ search: backend,
64
+ access,
50
65
  entityTransformers = [],
51
- queryBuilderClass = OpensearchQueryBuilder,
52
- queryBuilderOptions,
66
+ entityVisibilityFilter,
67
+ searchVisibilityFilter,
53
68
  } = opts;
54
- const queryBuilder = new queryBuilderClass(queryBuilderOptions);
69
+ const { capabilities } = backend;
70
+ const noFacets = Object.fromEntries(Object.keys(capabilities.facets).map((name) => [name, []]));
71
+
72
+ const violations = ({ searchType, sort }: SearchParams) => {
73
+ const found: Violation[] = [];
74
+ if (!Object.hasOwn(capabilities.searchTypes, searchType)) {
75
+ found.push({ field: '/searchType', message: `${searchType} search isn't supported`, value: searchType });
76
+ }
77
+ if (sort !== 'relevance' && !Object.hasOwn(capabilities.sorts, sort)) {
78
+ found.push({ field: '/sort', message: `${sort} isn't a declared sort`, value: sort });
79
+ }
80
+
81
+ return found;
82
+ };
83
+
55
84
  fastify.withTypeProvider<ZodTypeProvider>().post(
56
85
  '/search',
57
86
  {
@@ -60,140 +89,91 @@ const search: FastifyPluginAsync<SearchRouteOptions> = async (fastify, opts) =>
60
89
  },
61
90
  },
62
91
  async (request, reply) => {
63
- const { searchType, query, filters, boundingBox, geohashPrecision, limit, offset, sort, order } = request.body;
92
+ const { filters, violations: filterProblems } = splitFilters(capabilities.filters, request.body.filters);
93
+ const problems = [
94
+ ...violations(request.body),
95
+ ...filterProblems,
96
+ ...geoViolations(capabilities.geo, request.body),
97
+ ];
98
+ if (problems.length > 0) {
99
+ return reply.code(400).send(createValidationError('The request parameters are invalid', problems));
100
+ }
64
101
 
102
+ const { searchType, query, filters: _, boundingBox, geohashPrecision, ...paging } = request.body;
103
+
104
+ const searchQuery = normaliseQuery(searchType, query, capabilities.searchTypes.advanced?.fields ?? {});
105
+ if ('problem' in searchQuery) {
106
+ const { problem } = searchQuery;
107
+
108
+ return reply.code(422).send(createValidationError(problem, [{ field: 'query', message: problem }]));
109
+ }
110
+
111
+ const context = { request, fastify };
112
+ // Only the database query needs it, so the search needn't wait; awaited before any reply so its error wins
113
+ const visibilityWhere = Promise.resolve(entityVisibilityFilter(context));
114
+ visibilityWhere.catch(() => undefined);
115
+ const visibility = await searchVisibilityFilter(context);
116
+
117
+ const searchRequest: SearchRequest = {
118
+ ...searchQuery,
119
+ filters,
120
+ ...(boundingBox && { boundingBox }),
121
+ ...(geohashPrecision !== undefined && { geohashPrecision }),
122
+ ...paging,
123
+ visibility,
124
+ };
125
+
126
+ const start = performance.now();
127
+ let result: SearchResult;
65
128
  try {
66
- const opensearchQuery: Search_Request = {
67
- index: 'entities',
68
- body: {
69
- query: queryBuilder.buildQuery(searchType, query, filters, boundingBox),
70
- aggs: queryBuilder.buildAggregations(geohashPrecision, boundingBox),
71
- highlight: {
72
- fields: {
73
- name: {},
74
- description: {},
75
- },
76
- },
77
- sort: queryBuilder.buildSort(sort, order),
78
- from: offset,
79
- size: limit,
80
- },
81
- };
82
- fastify.log.debug(opensearchQuery);
83
- const response = await opensearch.search(opensearchQuery);
84
-
85
- if (!response.body?.hits?.hits) {
86
- throw new Error('Invalid search response: missing hits data');
129
+ result = await backend.search(searchRequest, context);
130
+ } catch (error) {
131
+ await visibilityWhere.catch((visibilityError: unknown) => {
132
+ fastify.log.error({ err: error }, 'Search backend failed');
133
+ throw visibilityError;
134
+ });
135
+ if (!(error instanceof SearchQueryError)) {
136
+ throw error;
87
137
  }
88
138
 
89
- const entityIds = response.body.hits.hits.map((hit) => hit._source?.id as string | undefined).filter(Boolean);
139
+ fastify.log.warn(`Invalid search query: ${error.message}`);
90
140
 
91
- const dbEntities = await prisma.entity.findMany({
92
- where: {
93
- id: {
94
- in: entityIds,
95
- },
96
- },
97
- include: { file: { select: { id: true } } },
98
- });
141
+ return reply.code(422).send(createError(ERROR_CODES.VALIDATION_ERROR, error.message));
142
+ }
143
+ const searchTime = Math.round(performance.now() - start);
99
144
 
100
- const entityMap = new Map(dbEntities.map((entity) => [entity.id, entity]));
101
-
102
- // Resolve memberOf and rootCollection references
103
- const refMap = await resolveEntityReferences(dbEntities, prisma);
104
-
105
- const entities = await Promise.all(
106
- response.body.hits.hits.map(async (hit) => {
107
- if (!hit._source?.id) {
108
- throw new Error('Missing id in search hit');
109
- }
110
-
111
- const dbEntity = entityMap.get(hit._source.id);
112
- if (!dbEntity) {
113
- fastify.log.warn(`Entity ${hit._source.id} found in OpenSearch but not in database`);
114
-
115
- return null;
116
- }
117
-
118
- const base = baseEntityTransformer(dbEntity);
119
- const standardEntity = {
120
- ...base,
121
- memberOf: base.memberOf ? (refMap.get(base.memberOf) ?? null) : null,
122
- rootCollection: base.rootCollection ? (refMap.get(base.rootCollection) ?? null) : null,
123
- };
124
- const authorisedEntity = await accessTransformer(standardEntity, { request, fastify });
125
-
126
- let result = authorisedEntity;
127
- for (const transformer of entityTransformers) {
128
- result = await transformer(result, { request, fastify });
129
- }
130
-
131
- // Add search-specific metadata
132
- return {
133
- ...(result as Record<string, unknown>),
134
- searchExtra: {
135
- score: hit._score,
136
- highlight: hit.highlight,
137
- },
138
- };
139
- }),
140
- ).then((results) => results.filter(Boolean));
141
-
142
- const facets: Record<string, Array<{ name: string; count: number }>> = {};
143
- if (response.body.aggregations) {
144
- Object.keys(response.body.aggregations).forEach((key) => {
145
- if (key !== 'geohash_grid') {
146
- const agg = response.body.aggregations?.[key] as MultiBucketAggregateBaseFiltersBucket;
147
- if (agg?.buckets && Array.isArray(agg.buckets)) {
148
- facets[key] = agg.buckets.map((bucket) => ({
149
- name: bucket.key,
150
- count: bucket.doc_count,
151
- }));
152
- }
153
- }
154
- });
155
- }
145
+ const dbEntities = await prisma.entity.findMany({
146
+ where: {
147
+ AND: [{ id: { in: result.hits.map(({ id }) => id) } }, await visibilityWhere],
148
+ },
149
+ });
156
150
 
157
- let geohashGrid: Record<string, number> | undefined;
158
- if (response.body.aggregations?.geohash_grid) {
159
- const geohashAgg = response.body.aggregations.geohash_grid as MultiBucketAggregateBaseFiltersBucket;
160
- if (geohashAgg?.buckets && Array.isArray(geohashAgg.buckets)) {
161
- geohashAgg.buckets.forEach((bucket) => {
162
- geohashGrid ||= {};
163
- geohashGrid[bucket.key] = bucket.doc_count;
164
- });
165
- }
166
- }
151
+ const authorisedEntities = await authoriseEntities(dbEntities, prisma, access, context);
152
+ const entityMap = new Map(authorisedEntities.map((entity, i) => [entity.id, { entity, row: dbEntities[i] }]));
167
153
 
168
- /* v8 ignore next 3 -- Not sure how to force opensearch to hit the other path -- @preserve */
169
- const total =
170
- typeof response.body.hits.total === 'number'
171
- ? response.body.hits.total
172
- : response.body.hits.total?.value || 0;
173
-
174
- const result = {
175
- total,
176
- searchTime: response.body.took,
177
- entities,
178
- facets: Object.keys(facets).length > 0 ? facets : undefined,
179
- geohashGrid,
180
- };
181
-
182
- return result;
183
- } catch (error) {
184
- const err = error as Error;
154
+ const entities = await Promise.all(
155
+ result.hits.map(async ({ id, searchExtra }) => {
156
+ const found = entityMap.get(id);
157
+ if (!found) {
158
+ fastify.log.warn(`Entity ${id} found by the search backend but missing or not visible in database`);
185
159
 
186
- // OpenSearch returns 400 for malformed queries (e.g. invalid query_string syntax)
187
- if ('statusCode' in err && (err as { statusCode: number }).statusCode === 400) {
188
- fastify.log.warn(`Invalid search query: ${err.message}`);
160
+ return null;
161
+ }
189
162
 
190
- return reply.code(422).send(createInvalidRequestError(err.message));
191
- }
163
+ const transformed = await applyTransformers(found.entity, entityTransformers, context, found.row);
164
+ const extra = visibleExtra(searchExtra, found.entity.access);
192
165
 
193
- fastify.log.error(`Search error: ${err.message}`);
166
+ return extra === undefined ? transformed : { ...(transformed as object), searchExtra: extra };
167
+ }),
168
+ ).then((results) => results.filter((entity) => entity !== null));
194
169
 
195
- return reply.code(500).send(createInternalError('Search failed'));
196
- }
170
+ return {
171
+ total: result.total,
172
+ searchTime,
173
+ entities,
174
+ facets: { ...noFacets, ...result.facets },
175
+ ...(result.geohashGrid && { geohashGrid: result.geohashGrid }),
176
+ };
197
177
  },
198
178
  );
199
179
  };
@@ -0,0 +1,258 @@
1
+ import { type AdvancedQuery, type SearchQuery, SearchQueryError } from './backend.js';
2
+ import { dateSpan } from './fields.js';
3
+
4
+ /** In code points, for basic and advanced queries */
5
+ const MAX_QUERY_LENGTH = 1024;
6
+ const MAX_DEPTH = 10;
7
+ /** Leaf clauses, counted after field groups are rewritten */
8
+ const MAX_CLAUSES = 64;
9
+
10
+ class InvalidQueryError extends Error {
11
+ override name = 'InvalidQueryError';
12
+ }
13
+
14
+ type Token = {
15
+ kind: '(' | ')' | ':' | 'phrase' | 'word' | 'AND' | 'OR' | 'NOT' | 'end';
16
+ text: string;
17
+ /** 1-based, in code points */
18
+ position: number;
19
+ };
20
+
21
+ const OPERATORS: readonly string[] = ['AND', 'OR', 'NOT'];
22
+ const PUNCTUATION: readonly string[] = ['(', ')', ':'];
23
+
24
+ const isSpace = (char: string) => /\s/u.test(char);
25
+ const endsWord = (char: string) => isSpace(char) || PUNCTUATION.includes(char) || char === '"';
26
+
27
+ const tokenise = (chars: string[]) => {
28
+ const tokens: Token[] = [];
29
+ let at = 0;
30
+
31
+ while (at < chars.length) {
32
+ const char = chars[at];
33
+ const position = at + 1;
34
+
35
+ if (isSpace(char)) {
36
+ at++;
37
+ } else if (PUNCTUATION.includes(char)) {
38
+ tokens.push({ kind: char as Token['kind'], text: char, position });
39
+ at++;
40
+ } else if (char === '"') {
41
+ const close = chars.indexOf('"', at + 1);
42
+ if (close === -1) {
43
+ throw new InvalidQueryError(
44
+ `Expected a closing " for the phrase at position ${position}, found the end of the query`,
45
+ );
46
+ }
47
+ tokens.push({ kind: 'phrase', text: chars.slice(at + 1, close).join(''), position });
48
+ at = close + 1;
49
+ } else {
50
+ const start = at;
51
+ while (at < chars.length && !endsWord(chars[at])) {
52
+ at++;
53
+ }
54
+ const text = chars.slice(start, at).join('');
55
+ tokens.push({ kind: OPERATORS.includes(text) ? (text as Token['kind']) : 'word', text, position });
56
+ }
57
+ }
58
+
59
+ tokens.push({ kind: 'end', text: '', position: chars.length + 1 });
60
+
61
+ return tokens;
62
+ };
63
+
64
+ const describe = ({ kind, text }: Token) => {
65
+ switch (kind) {
66
+ case 'end':
67
+ return 'the end of the query';
68
+ case 'phrase':
69
+ return `the phrase "${text}"`;
70
+ case 'word':
71
+ return `"${text}"`;
72
+ default:
73
+ return kind.length === 1 ? `"${kind}"` : kind;
74
+ }
75
+ };
76
+
77
+ const combine = (type: 'and' | 'or', parts: AdvancedQuery[]): AdvancedQuery =>
78
+ parts.length === 1
79
+ ? parts[0]
80
+ : { type, clauses: parts.flatMap((part) => (part.type === type ? part.clauses : [part])) };
81
+
82
+ const parse = (query: string, fields: Record<string, unknown>): AdvancedQuery => {
83
+ const tokens = tokenise([...query]);
84
+ let at = 0;
85
+
86
+ const peek = (offset = 0) => tokens[at + offset];
87
+ const fail = (expected: string): never => {
88
+ throw new InvalidQueryError(`Expected ${expected} at position ${peek().position}, found ${describe(peek())}`);
89
+ };
90
+ const expect = (kind: Token['kind'], expected: string) => {
91
+ if (peek().kind !== kind) {
92
+ fail(expected);
93
+ }
94
+ at++;
95
+ };
96
+
97
+ // `field` is set inside a field group, where clauses are values rather than field matches
98
+ const parseOr = (field?: string): AdvancedQuery => {
99
+ const parts = [parseAnd(field)];
100
+ while (peek().kind === 'OR') {
101
+ at++;
102
+ parts.push(parseAnd(field));
103
+ }
104
+
105
+ return combine('or', parts);
106
+ };
107
+
108
+ const parseAnd = (field?: string): AdvancedQuery => {
109
+ const parts = [parseUnary(field)];
110
+ while (peek().kind === 'AND') {
111
+ at++;
112
+ parts.push(parseUnary(field));
113
+ }
114
+
115
+ return combine('and', parts);
116
+ };
117
+
118
+ const parseUnary = (field?: string): AdvancedQuery => {
119
+ if (peek().kind === 'NOT') {
120
+ at++;
121
+
122
+ return { type: 'not', clause: parseUnary(field) };
123
+ }
124
+
125
+ return parsePrimary(field);
126
+ };
127
+
128
+ const parseBracketed = (field?: string) => {
129
+ at++;
130
+ const clause = parseOr(field);
131
+ expect(')', 'AND, OR or ")"');
132
+
133
+ return clause;
134
+ };
135
+
136
+ const parsePrimary = (field?: string): AdvancedQuery => {
137
+ if (peek().kind === '(') {
138
+ return parseBracketed(field);
139
+ }
140
+ if (field !== undefined) {
141
+ return parseValue(field);
142
+ }
143
+
144
+ const name = peek();
145
+ if (name.kind !== 'word') {
146
+ fail('a field name');
147
+ }
148
+ at++;
149
+ expect(':', `":" after the field name "${name.text}"`);
150
+ if (!Object.hasOwn(fields, name.text)) {
151
+ throw new InvalidQueryError(`"${name.text}" at position ${name.position} isn't a declared advanced field`);
152
+ }
153
+
154
+ return peek().kind === '(' ? parseBracketed(name.text) : parseValue(name.text);
155
+ };
156
+
157
+ const startsValueWord = () => peek().kind === 'word' && peek(1).kind !== ':';
158
+
159
+ const parseValue = (field: string): AdvancedQuery => {
160
+ if (peek().kind === 'phrase') {
161
+ const { text } = peek();
162
+ at++;
163
+
164
+ return { type: 'match', field, value: text, phrase: true };
165
+ }
166
+
167
+ if (!startsValueWord()) {
168
+ fail(`a value for "${field}"`);
169
+ }
170
+ const words: string[] = [];
171
+ while (startsValueWord()) {
172
+ words.push(peek().text);
173
+ at++;
174
+ }
175
+
176
+ return { type: 'match', field, value: words.join(' '), phrase: false };
177
+ };
178
+
179
+ const tree = parseOr();
180
+ expect('end', 'AND, OR or the end of the query');
181
+
182
+ return tree;
183
+ };
184
+
185
+ const depthOf = (query: AdvancedQuery): number => {
186
+ switch (query.type) {
187
+ case 'and':
188
+ case 'or':
189
+ return 1 + Math.max(...query.clauses.map(depthOf));
190
+ case 'not':
191
+ return 1 + depthOf(query.clause);
192
+ default:
193
+ return 0;
194
+ }
195
+ };
196
+
197
+ const clausesIn = (query: AdvancedQuery): number => {
198
+ switch (query.type) {
199
+ case 'and':
200
+ case 'or':
201
+ return query.clauses.reduce((sum, clause) => sum + clausesIn(clause), 0);
202
+ case 'not':
203
+ return clausesIn(query.clause);
204
+ default:
205
+ return 1;
206
+ }
207
+ };
208
+
209
+ const parseWithinCaps = (query: string, advancedFields: Record<string, unknown>): AdvancedQuery => {
210
+ const tree = parse(query, advancedFields);
211
+ if (depthOf(tree) > MAX_DEPTH) {
212
+ throw new InvalidQueryError(`The query nests deeper than the limit of ${MAX_DEPTH} levels`);
213
+ }
214
+ if (clausesIn(tree) > MAX_CLAUSES) {
215
+ throw new InvalidQueryError(`The query has more than the limit of ${MAX_CLAUSES} clauses`);
216
+ }
217
+
218
+ return tree;
219
+ };
220
+
221
+ /**
222
+ * Check a query against the caps and parse an advanced one into a tree over `advancedFields`
223
+ * A query arocapi refuses gives the problem instead
224
+ */
225
+ export const normaliseQuery = (
226
+ searchType: 'basic' | 'advanced',
227
+ query: string,
228
+ advancedFields: Record<string, unknown>,
229
+ ): SearchQuery | { problem: string } => {
230
+ if ([...query].length > MAX_QUERY_LENGTH) {
231
+ return { problem: `The query is longer than the limit of ${MAX_QUERY_LENGTH} characters` };
232
+ }
233
+ if (searchType === 'basic') {
234
+ return { searchType, query };
235
+ }
236
+ if (query.trim() === '') {
237
+ return { searchType, query: { type: 'all' } };
238
+ }
239
+
240
+ try {
241
+ return { searchType, query: parseWithinCaps(query, advancedFields) };
242
+ } catch (error) {
243
+ // Only InvalidQueryError is thrown: the parser is pure and its recursion is bounded by the length cap
244
+ return { problem: (error as InvalidQueryError).message };
245
+ }
246
+ };
247
+
248
+ /**
249
+ * The UTC instants an advanced match on a date field covers, throwing a SearchQueryError for a value that isn't a date
250
+ */
251
+ export const matchedDateSpan = (field: string, value: string) => {
252
+ const span = dateSpan(value);
253
+ if (!span) {
254
+ throw new SearchQueryError(`"${field}" is a date field: "${value}" isn't a date`);
255
+ }
256
+
257
+ return span;
258
+ };
@@ -0,0 +1,91 @@
1
+ import type { RequestContext } from '../types/context.js';
2
+ import type { GeoPoint, SearchDocument } from './fields.js';
3
+ import type { SearchVisibilityWhere } from './visibility.js';
4
+
5
+ type Labelled = { label?: string };
6
+
7
+ /**
8
+ * What a search backend publishes in `/capabilities` and arocapi validates requests against
9
+ */
10
+ export type BackendSearchCapability = {
11
+ filters: Record<string, Labelled & { type: 'string' | 'date' | 'number' | 'boolean' }>;
12
+ facets: Record<string, Labelled>;
13
+ sorts: Record<string, Labelled>;
14
+ searchTypes: { basic: Record<string, never>; advanced?: { fields: Record<string, Labelled> } };
15
+ geo: { supported: boolean };
16
+ };
17
+
18
+ /**
19
+ * A parsed advanced query
20
+ */
21
+ export type AdvancedQuery =
22
+ | { type: 'and' | 'or'; clauses: AdvancedQuery[] }
23
+ | { type: 'not'; clause: AdvancedQuery }
24
+ | { type: 'match'; field: string; value: string; phrase: boolean }
25
+ | { type: 'all' };
26
+
27
+ /** Date bounds are UTC instants */
28
+ export type RangeBound = number | Date;
29
+
30
+ export type RangeFilter = { gte?: RangeBound; lte?: RangeBound };
31
+
32
+ export type BoundingBox = { topRight: GeoPoint; bottomLeft: GeoPoint };
33
+
34
+ export type SearchQuery = { searchType: 'basic'; query: string } | { searchType: 'advanced'; query: AdvancedQuery };
35
+
36
+ /**
37
+ * A validated, normalised search, as arocapi hands it to a search backend
38
+ */
39
+ export type SearchRequest = SearchQuery & {
40
+ filters: {
41
+ /** Field name → values, any of which matches */
42
+ terms: Record<string, string[]>;
43
+ /** Field name → ranges, any of which matches; date and number filters always arrive here, exact values as ranges */
44
+ ranges: Record<string, RangeFilter[]>;
45
+ };
46
+ boundingBox?: BoundingBox;
47
+ geohashPrecision?: number;
48
+ limit: number;
49
+ offset: number;
50
+ /** A declared sort, or `relevance`: best match first, stable for an empty query */
51
+ sort: string;
52
+ order: 'asc' | 'desc';
53
+ /** The resolved `searchVisibilityFilter`, which the backend checks and applies to hits, total and facets */
54
+ visibility: SearchVisibilityWhere;
55
+ };
56
+
57
+ export type SearchExtra = { score?: number; highlight?: Record<string, string[]> };
58
+
59
+ export type FacetBucket = { name: string; count: number };
60
+
61
+ /** The most geohash cells a reference backend returns: those with the most hits */
62
+ export const GEOHASH_GRID_CELLS = 10_000;
63
+
64
+ export type SearchResult = {
65
+ /** Exact */
66
+ total: number;
67
+ /** Ordered */
68
+ hits: { id: string; searchExtra?: SearchExtra }[];
69
+ facets?: Record<string, FacetBucket[]>;
70
+ geohashGrid?: Record<string, number>;
71
+ };
72
+
73
+ /**
74
+ * A search backend: arocapi validates each request against `capabilities`, the backend only searches
75
+ */
76
+ export interface SearchBackend {
77
+ /** What `/capabilities` publishes and arocapi validates requests against */
78
+ readonly capabilities: BackendSearchCapability;
79
+ search(request: SearchRequest, context: RequestContext): Promise<SearchResult>;
80
+ /** Upsert search documents; each entity must exist first */
81
+ index(docs: SearchDocument[], options?: { tx?: unknown }): Promise<void>;
82
+ /** Remove search documents, ignoring ids that aren't indexed */
83
+ remove(ids: string[]): Promise<void>;
84
+ }
85
+
86
+ /**
87
+ * Thrown by a search backend for a query it can't run; arocapi answers 422 with its message
88
+ */
89
+ export class SearchQueryError extends Error {
90
+ override name = 'SearchQueryError';
91
+ }