arocapi 4.0.1 → 5.0.0-beta.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (242) hide show
  1. package/LICENSE.md +1 -1
  2. package/README.md +37 -781
  3. package/dist/app.d.ts +38 -21
  4. package/dist/app.js +63 -45
  5. package/dist/app.js.map +1 -1
  6. package/dist/routes/capabilities.d.ts +7 -0
  7. package/dist/routes/capabilities.js +13 -0
  8. package/dist/routes/capabilities.js.map +1 -0
  9. package/dist/routes/entities.d.ts +5 -2
  10. package/dist/routes/entities.js +38 -58
  11. package/dist/routes/entities.js.map +1 -1
  12. package/dist/routes/entity.d.ts +7 -2
  13. package/dist/routes/entity.js +14 -30
  14. package/dist/routes/entity.js.map +1 -1
  15. package/dist/routes/file.d.ts +8 -2
  16. package/dist/routes/file.js +38 -73
  17. package/dist/routes/file.js.map +1 -1
  18. package/dist/routes/files.d.ts +5 -2
  19. package/dist/routes/files.js +25 -40
  20. package/dist/routes/files.js.map +1 -1
  21. package/dist/routes/metadata.d.ts +15 -0
  22. package/dist/routes/metadata.js +38 -0
  23. package/dist/routes/metadata.js.map +1 -0
  24. package/dist/routes/search.d.ts +8 -7
  25. package/dist/routes/search.js +91 -120
  26. package/dist/routes/search.js.map +1 -1
  27. package/dist/search/advanced.d.ts +8 -0
  28. package/dist/search/advanced.js +198 -0
  29. package/dist/search/advanced.js.map +1 -0
  30. package/dist/search/backend.d.ts +95 -0
  31. package/dist/search/backend.js +5 -0
  32. package/dist/search/backend.js.map +1 -0
  33. package/dist/search/fields.d.ts +63 -0
  34. package/dist/search/fields.js +203 -0
  35. package/dist/search/fields.js.map +1 -0
  36. package/dist/search/opensearch/advanced.d.ts +4 -0
  37. package/dist/search/opensearch/advanced.js +28 -0
  38. package/dist/search/opensearch/advanced.js.map +1 -0
  39. package/dist/search/opensearch/opensearch.d.ts +16 -0
  40. package/dist/search/opensearch/opensearch.js +187 -0
  41. package/dist/search/opensearch/opensearch.js.map +1 -0
  42. package/dist/search/opensearch/visibility.d.ts +3 -0
  43. package/dist/search/opensearch/visibility.js +27 -0
  44. package/dist/search/opensearch/visibility.js.map +1 -0
  45. package/dist/search/postgres/advanced.d.ts +7 -0
  46. package/dist/search/postgres/advanced.js +54 -0
  47. package/dist/search/postgres/advanced.js.map +1 -0
  48. package/dist/search/postgres/facets.d.ts +2 -0
  49. package/dist/search/postgres/facets.js +27 -0
  50. package/dist/search/postgres/facets.js.map +1 -0
  51. package/dist/search/postgres/geo.d.ts +6 -0
  52. package/dist/search/postgres/geo.js +42 -0
  53. package/dist/search/postgres/geo.js.map +1 -0
  54. package/dist/search/postgres/postgres.d.ts +19 -0
  55. package/dist/search/postgres/postgres.js +157 -0
  56. package/dist/search/postgres/postgres.js.map +1 -0
  57. package/dist/search/postgres/values.d.ts +8 -0
  58. package/dist/search/postgres/values.js +20 -0
  59. package/dist/search/postgres/values.js.map +1 -0
  60. package/dist/search/postgres/visibility.d.ts +4 -0
  61. package/dist/search/postgres/visibility.js +38 -0
  62. package/dist/search/postgres/visibility.js.map +1 -0
  63. package/dist/search/requestFilters.d.ts +39 -0
  64. package/dist/search/requestFilters.js +125 -0
  65. package/dist/search/requestFilters.js.map +1 -0
  66. package/dist/search/visibility.d.ts +51 -0
  67. package/dist/search/visibility.js +100 -0
  68. package/dist/search/visibility.js.map +1 -0
  69. package/dist/storage/backend.d.ts +58 -0
  70. package/dist/storage/backend.js +2 -0
  71. package/dist/storage/backend.js.map +1 -0
  72. package/dist/storage/deliver.d.ts +12 -0
  73. package/dist/storage/deliver.js +34 -0
  74. package/dist/storage/deliver.js.map +1 -0
  75. package/dist/storage/directory/directory.d.ts +6 -0
  76. package/dist/storage/directory/directory.js +24 -0
  77. package/dist/storage/directory/directory.js.map +1 -0
  78. package/dist/storage/entity-meta/entity-meta.d.ts +5 -0
  79. package/dist/storage/entity-meta/entity-meta.js +14 -0
  80. package/dist/storage/entity-meta/entity-meta.js.map +1 -0
  81. package/dist/storage/location.d.ts +19 -0
  82. package/dist/storage/location.js +36 -0
  83. package/dist/storage/location.js.map +1 -0
  84. package/dist/storage/ocfl/ocfl.d.ts +6 -0
  85. package/dist/storage/ocfl/ocfl.js +45 -0
  86. package/dist/storage/ocfl/ocfl.js.map +1 -0
  87. package/dist/storage/results.d.ts +7 -0
  88. package/dist/storage/results.js +22 -0
  89. package/dist/storage/results.js.map +1 -0
  90. package/dist/storage/s3/s3.d.ts +10 -0
  91. package/dist/storage/s3/s3.js +47 -0
  92. package/dist/storage/s3/s3.js.map +1 -0
  93. package/dist/transformers/default.d.ts +11 -22
  94. package/dist/transformers/default.js +41 -15
  95. package/dist/transformers/default.js.map +1 -1
  96. package/dist/types/access.d.ts +9 -0
  97. package/dist/types/access.js +2 -0
  98. package/dist/types/access.js.map +1 -0
  99. package/dist/types/context.d.ts +6 -0
  100. package/dist/types/context.js +2 -0
  101. package/dist/types/context.js.map +1 -0
  102. package/dist/types/transformers.d.ts +4 -10
  103. package/dist/types/visibility.d.ts +6 -0
  104. package/dist/types/visibility.js +2 -0
  105. package/dist/types/visibility.js.map +1 -0
  106. package/dist/utils/access.d.ts +17 -0
  107. package/dist/utils/access.js +41 -0
  108. package/dist/utils/access.js.map +1 -0
  109. package/dist/utils/errors.d.ts +19 -10
  110. package/dist/utils/errors.js +44 -8
  111. package/dist/utils/errors.js.map +1 -1
  112. package/dist/utils/headers.d.ts +1 -1
  113. package/dist/utils/headers.js +10 -1
  114. package/dist/utils/headers.js.map +1 -1
  115. package/dist/utils/validation.d.ts +2 -0
  116. package/dist/utils/validation.js +8 -0
  117. package/dist/utils/validation.js.map +1 -0
  118. package/dist/utils/visibility.d.ts +18 -0
  119. package/dist/utils/visibility.js +16 -0
  120. package/dist/utils/visibility.js.map +1 -0
  121. package/package.json +84 -45
  122. package/prisma/models/entity.prisma +9 -8
  123. package/prisma/models/entitySearch.prisma +21 -0
  124. package/prisma/models/file.prisma +3 -3
  125. package/src/app.ts +169 -102
  126. package/src/routes/capabilities.ts +22 -0
  127. package/src/routes/entities.ts +49 -68
  128. package/src/routes/entity.ts +22 -36
  129. package/src/routes/file.ts +54 -104
  130. package/src/routes/files.ts +32 -48
  131. package/src/routes/metadata.ts +67 -0
  132. package/src/routes/search.ts +132 -152
  133. package/src/search/advanced.ts +258 -0
  134. package/src/search/backend.ts +91 -0
  135. package/src/search/fields.ts +320 -0
  136. package/src/search/opensearch/advanced.ts +36 -0
  137. package/src/search/opensearch/opensearch.ts +290 -0
  138. package/src/search/opensearch/visibility.ts +38 -0
  139. package/src/search/postgres/advanced.ts +76 -0
  140. package/src/search/postgres/facets.ts +34 -0
  141. package/src/search/postgres/geo.ts +58 -0
  142. package/src/search/postgres/postgres.ts +232 -0
  143. package/src/search/postgres/values.ts +33 -0
  144. package/src/search/postgres/visibility.ts +54 -0
  145. package/src/search/requestFilters.ts +178 -0
  146. package/src/search/visibility.ts +193 -0
  147. package/src/storage/backend.ts +75 -0
  148. package/src/storage/deliver.ts +65 -0
  149. package/src/storage/directory/directory.ts +39 -0
  150. package/src/storage/entity-meta/entity-meta.ts +25 -0
  151. package/src/storage/location.ts +78 -0
  152. package/src/storage/ocfl/ocfl.ts +71 -0
  153. package/src/storage/results.ts +32 -0
  154. package/src/storage/s3/s3.ts +79 -0
  155. package/src/transformers/default.ts +115 -85
  156. package/src/types/access.ts +25 -0
  157. package/src/types/context.ts +15 -0
  158. package/src/types/transformers.ts +8 -34
  159. package/src/types/visibility.ts +21 -0
  160. package/src/utils/access.ts +80 -0
  161. package/src/utils/errors.ts +68 -11
  162. package/src/utils/headers.ts +22 -2
  163. package/src/utils/validation.ts +11 -0
  164. package/src/utils/visibility.ts +56 -0
  165. package/dist/app.test.d.ts +0 -1
  166. package/dist/app.test.js +0 -89
  167. package/dist/app.test.js.map +0 -1
  168. package/dist/express.d.ts +0 -4
  169. package/dist/express.js +0 -14
  170. package/dist/express.js.map +0 -1
  171. package/dist/index.dev.d.ts +0 -1
  172. package/dist/index.dev.js +0 -156
  173. package/dist/index.dev.js.map +0 -1
  174. package/dist/routes/crate.d.ts +0 -9
  175. package/dist/routes/crate.js +0 -82
  176. package/dist/routes/crate.js.map +0 -1
  177. package/dist/routes/crate.test.d.ts +0 -1
  178. package/dist/routes/crate.test.js +0 -342
  179. package/dist/routes/crate.test.js.map +0 -1
  180. package/dist/routes/entities.test.d.ts +0 -1
  181. package/dist/routes/entities.test.js +0 -273
  182. package/dist/routes/entities.test.js.map +0 -1
  183. package/dist/routes/entity.test.d.ts +0 -1
  184. package/dist/routes/entity.test.js +0 -130
  185. package/dist/routes/entity.test.js.map +0 -1
  186. package/dist/routes/file.test.d.ts +0 -1
  187. package/dist/routes/file.test.js +0 -315
  188. package/dist/routes/file.test.js.map +0 -1
  189. package/dist/routes/files.test.d.ts +0 -1
  190. package/dist/routes/files.test.js +0 -222
  191. package/dist/routes/files.test.js.map +0 -1
  192. package/dist/routes/search.test.d.ts +0 -1
  193. package/dist/routes/search.test.js +0 -780
  194. package/dist/routes/search.test.js.map +0 -1
  195. package/dist/test/helpers/fastify.d.ts +0 -9
  196. package/dist/test/helpers/fastify.js +0 -18
  197. package/dist/test/helpers/fastify.js.map +0 -1
  198. package/dist/test/integration.setup.d.ts +0 -29
  199. package/dist/test/integration.setup.js +0 -188
  200. package/dist/test/integration.setup.js.map +0 -1
  201. package/dist/test/integration.test.d.ts +0 -1
  202. package/dist/test/integration.test.js +0 -396
  203. package/dist/test/integration.test.js.map +0 -1
  204. package/dist/transformers/default.test.d.ts +0 -1
  205. package/dist/transformers/default.test.js +0 -163
  206. package/dist/transformers/default.test.js.map +0 -1
  207. package/dist/transformers/transformer.test.d.ts +0 -1
  208. package/dist/transformers/transformer.test.js +0 -187
  209. package/dist/transformers/transformer.test.js.map +0 -1
  210. package/dist/types/fileHandlers.d.ts +0 -41
  211. package/dist/types/fileHandlers.js +0 -2
  212. package/dist/types/fileHandlers.js.map +0 -1
  213. package/dist/utils/errors.test.d.ts +0 -1
  214. package/dist/utils/errors.test.js +0 -66
  215. package/dist/utils/errors.test.js.map +0 -1
  216. package/dist/utils/queryBuilder.d.ts +0 -35
  217. package/dist/utils/queryBuilder.js +0 -98
  218. package/dist/utils/queryBuilder.js.map +0 -1
  219. package/prisma.config.ts +0 -9
  220. package/src/app.test.ts +0 -119
  221. package/src/express.ts +0 -21
  222. package/src/index.dev.ts +0 -200
  223. package/src/reset.d.ts +0 -1
  224. package/src/routes/__snapshots__/entities.test.ts.snap +0 -96
  225. package/src/routes/__snapshots__/entity.test.ts.snap +0 -74
  226. package/src/routes/__snapshots__/search.test.ts.snap +0 -165
  227. package/src/routes/crate.test.ts +0 -423
  228. package/src/routes/crate.ts +0 -123
  229. package/src/routes/entities.test.ts +0 -320
  230. package/src/routes/entity.test.ts +0 -160
  231. package/src/routes/file.test.ts +0 -395
  232. package/src/routes/files.test.ts +0 -278
  233. package/src/routes/search.test.ts +0 -903
  234. package/src/test/__snapshots__/integration.test.ts.snap +0 -18
  235. package/src/test/helpers/fastify.ts +0 -27
  236. package/src/test/integration.setup.ts +0 -210
  237. package/src/test/integration.test.ts +0 -482
  238. package/src/transformers/default.test.ts +0 -184
  239. package/src/transformers/transformer.test.ts +0 -252
  240. package/src/types/fileHandlers.ts +0 -65
  241. package/src/utils/errors.test.ts +0 -75
  242. package/src/utils/queryBuilder.ts +0 -132
@@ -0,0 +1,320 @@
1
+ import type { BackendSearchCapability } from './backend.js';
2
+
3
+ export type FieldType = 'text' | 'keyword' | 'date' | 'number' | 'boolean' | 'geo';
4
+
5
+ export type FieldDeclaration = {
6
+ type: FieldType;
7
+ label?: string;
8
+ /** Text fields only: 1–4, default 1 */
9
+ weight?: number;
10
+ /** Default true for text fields */
11
+ basic?: boolean;
12
+ advanced?: boolean;
13
+ filter?: boolean;
14
+ /** Implies `filter`. Default size 20 */
15
+ facet?: boolean | { size: number };
16
+ sort?: boolean;
17
+ };
18
+
19
+ export type FieldDeclarations = Record<string, FieldDeclaration>;
20
+
21
+ const FIELD_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/;
22
+ const RESERVED: readonly string[] = ['id', 'AND', 'OR', 'NOT'];
23
+ const FIELD_TYPES: readonly string[] = ['text', 'keyword', 'date', 'number', 'boolean', 'geo'];
24
+
25
+ const isPositiveInteger = (value: number) => Number.isInteger(value) && value > 0;
26
+
27
+ const checkDeclaration = (name: string, declaration: FieldDeclaration) => {
28
+ if (!FIELD_NAME.test(name)) {
29
+ return `must match ${FIELD_NAME}`;
30
+ }
31
+ if (RESERVED.includes(name)) {
32
+ return 'is reserved';
33
+ }
34
+
35
+ const { type, weight, facet } = declaration;
36
+ if (!FIELD_TYPES.includes(type)) {
37
+ return `has unknown type "${type}"`;
38
+ }
39
+ if (type === 'geo') {
40
+ const role = (['basic', 'advanced', 'filter', 'facet', 'sort'] as const).find((r) => declaration[r]);
41
+ if (role) {
42
+ return `is a geo field, which can't have the ${role} role`;
43
+ }
44
+ }
45
+ if (name === 'relevance' && declaration.sort) {
46
+ return "can't be a sort field, as relevance is always a sort";
47
+ }
48
+ if (weight !== undefined && (type !== 'text' || !isPositiveInteger(weight) || weight > 4)) {
49
+ return 'has a bad weight: only a text field has one, from 1 to 4';
50
+ }
51
+ if (typeof facet === 'object' && !isPositiveInteger(facet.size)) {
52
+ return 'has a bad facet size: it must be a positive integer';
53
+ }
54
+ if (facet && type === 'number') {
55
+ return "is a number field, which can't be a facet";
56
+ }
57
+ if (declaration.basic && type !== 'text' && type !== 'keyword') {
58
+ return `is a ${type} field, which can't be in basic search`;
59
+ }
60
+
61
+ return undefined;
62
+ };
63
+
64
+ /**
65
+ * Throw on a field declaration a search backend can't hold
66
+ */
67
+ const validateFields = (fields: FieldDeclarations) => {
68
+ let geoField: string | undefined;
69
+
70
+ for (const [name, declaration] of Object.entries(fields)) {
71
+ const problem = checkDeclaration(name, declaration);
72
+ if (problem) {
73
+ throw new Error(`Search field "${name}" ${problem}`);
74
+ }
75
+
76
+ if (declaration.type === 'geo') {
77
+ if (geoField) {
78
+ throw new Error(`Search field "${name}" is a second geo field, after "${geoField}": only one is allowed`);
79
+ }
80
+ geoField = name;
81
+ }
82
+ }
83
+ };
84
+
85
+ /**
86
+ * Derive a search backend's `/capabilities`-shaped capabilities from its field declarations
87
+ */
88
+ export const capabilitiesFromFields = (fields: FieldDeclarations): BackendSearchCapability => {
89
+ validateFields(fields);
90
+
91
+ const declared = Object.entries(fields);
92
+ const publish = <Published extends object>(
93
+ hasRole: (declaration: FieldDeclaration) => unknown,
94
+ describe: (declaration: FieldDeclaration) => Published = () => ({}) as Published,
95
+ ) =>
96
+ Object.fromEntries(
97
+ declared
98
+ .filter(([, declaration]) => hasRole(declaration))
99
+ .map(([name, declaration]) => [
100
+ name,
101
+ { ...describe(declaration), ...(declaration.label !== undefined && { label: declaration.label }) },
102
+ ]),
103
+ );
104
+
105
+ const advanced = publish(({ advanced }) => advanced);
106
+
107
+ return {
108
+ filters: publish(
109
+ ({ filter, facet }) => filter || facet,
110
+ ({ type }) => ({
111
+ type: type === 'text' || type === 'keyword' ? 'string' : (type as 'date' | 'number' | 'boolean'),
112
+ }),
113
+ ),
114
+ facets: publish(({ facet }) => facet),
115
+ sorts: publish(({ sort }) => sort),
116
+ searchTypes: { basic: {}, ...(Object.keys(advanced).length > 0 && { advanced: { fields: advanced } }) },
117
+ geo: { supported: declared.some(([, { type }]) => type === 'geo') },
118
+ };
119
+ };
120
+
121
+ const DEFAULT_FACET_SIZE = 20;
122
+
123
+ export const isBasic = ({ type, basic }: FieldDeclaration) => (type === 'text' ? basic !== false : Boolean(basic));
124
+
125
+ /**
126
+ * The declarations with their defaults applied, as a search backend builds its queries from them
127
+ */
128
+ export const resolveFields = (fields: FieldDeclarations) => {
129
+ const declared = Object.entries(fields);
130
+
131
+ return {
132
+ basic: declared
133
+ .filter(([, declaration]) => isBasic(declaration))
134
+ .map(([name, { type, weight = 1 }]) => ({ name, type, weight })),
135
+ facets: declared
136
+ .filter(([, { facet }]) => facet)
137
+ .map(([name, { type, facet }]) => ({
138
+ name,
139
+ type,
140
+ size: typeof facet === 'object' ? facet.size : DEFAULT_FACET_SIZE,
141
+ })),
142
+ geoField: declared.find(([, { type }]) => type === 'geo')?.[0],
143
+ };
144
+ };
145
+
146
+ /**
147
+ * The declarations with the implicit `id` field, which a search visibility filter can also use
148
+ */
149
+ export const withId = (fields: FieldDeclarations): FieldDeclarations => ({ ...fields, id: { type: 'keyword' } });
150
+
151
+ export type GeoPoint = { lat: number; lng: number };
152
+
153
+ type ScalarOf<Type extends FieldType> = Type extends 'number'
154
+ ? number
155
+ : Type extends 'boolean'
156
+ ? boolean
157
+ : Type extends 'geo'
158
+ ? GeoPoint
159
+ : string;
160
+
161
+ type ValueOf<Declaration extends FieldDeclaration> = Declaration['type'] extends 'geo'
162
+ ? GeoPoint
163
+ : Declaration extends { sort: true }
164
+ ? ScalarOf<Declaration['type']>
165
+ : ScalarOf<Declaration['type']> | ScalarOf<Declaration['type']>[];
166
+
167
+ /**
168
+ * One entity's values for the declared fields, as a consumer supplies it to a search backend
169
+ */
170
+ export type SearchDocument<Fields extends FieldDeclarations = FieldDeclarations> = { id: string } & {
171
+ -readonly [Name in keyof Fields]?: ValueOf<Fields[Name]>;
172
+ };
173
+
174
+ export type Scalar = string | number | boolean | Date;
175
+
176
+ /** A value as JSON, with dates as ISO strings */
177
+ export const toWire = (value: Scalar) => (value instanceof Date ? value.toISOString() : value);
178
+
179
+ /**
180
+ * A validated search document: dates are their first UTC instant, missing fields are left out and the geo field's
181
+ * value is the point
182
+ */
183
+ type StoredSearchDocument = { id: string; values: Record<string, Scalar | Scalar[]>; point?: GeoPoint };
184
+
185
+ const DATE = /^(\d{4})(?:-(\d{2})(?:-(\d{2})(?:[Tt ](\d{2}):(\d{2}):(\d{2})(?:\.(\d+))?([Zz]|[+-]\d{2}:\d{2})?)?)?)?$/;
186
+
187
+ const parseDate = (value: string) => {
188
+ const match = DATE.exec(value);
189
+ if (!match) {
190
+ return undefined;
191
+ }
192
+
193
+ const [, year, month = '01', day = '01', hour = '00', minute = '00', second = '00', fraction = '', offset = 'Z'] =
194
+ match;
195
+ const parts = [year, month, day, hour, minute, second].map(Number);
196
+
197
+ const date = new Date(0);
198
+ date.setUTCFullYear(parts[0], parts[1] - 1, parts[2]);
199
+ date.setUTCHours(parts[3], parts[4], parts[5], Number(fraction.padEnd(3, '0').slice(0, 3)));
200
+
201
+ const stored = [
202
+ date.getUTCFullYear(),
203
+ date.getUTCMonth() + 1,
204
+ date.getUTCDate(),
205
+ date.getUTCHours(),
206
+ date.getUTCMinutes(),
207
+ date.getUTCSeconds(),
208
+ ];
209
+ if (stored.some((part, index) => part !== parts[index])) {
210
+ return undefined;
211
+ }
212
+
213
+ if (offset.toUpperCase() !== 'Z') {
214
+ const [offsetHours, offsetMinutes] = offset.slice(1).split(':').map(Number);
215
+ if (offsetHours > 23 || offsetMinutes > 59) {
216
+ return undefined;
217
+ }
218
+ const sign = offset.startsWith('-') ? -1 : 1;
219
+ date.setTime(date.getTime() - sign * (offsetHours * 60 + offsetMinutes) * 60_000);
220
+ }
221
+
222
+ return date;
223
+ };
224
+
225
+ /**
226
+ * The UTC instants a date covers, as a match value: a year, month or day is the whole period, a date-time its instant
227
+ */
228
+ export const dateSpan = (value: string) => {
229
+ const first = parseDate(value);
230
+ if (!first) {
231
+ return undefined;
232
+ }
233
+
234
+ const [, , month, day, hour] = DATE.exec(value) as RegExpExecArray;
235
+ if (hour !== undefined) {
236
+ return { first, last: first };
237
+ }
238
+
239
+ const next = new Date(first);
240
+ if (day !== undefined) {
241
+ next.setUTCDate(next.getUTCDate() + 1);
242
+ } else if (month !== undefined) {
243
+ next.setUTCMonth(next.getUTCMonth() + 1);
244
+ } else {
245
+ next.setUTCFullYear(next.getUTCFullYear() + 1);
246
+ }
247
+
248
+ return { first, last: new Date(next.getTime() - 1) };
249
+ };
250
+
251
+ const isGeoPoint = (value: unknown): value is GeoPoint => {
252
+ if (typeof value !== 'object' || value === null || Object.keys(value).length !== 2) {
253
+ return false;
254
+ }
255
+ const { lat, lng } = value as Partial<GeoPoint>;
256
+
257
+ return typeof lat === 'number' && typeof lng === 'number' && Math.abs(lat) <= 90 && Math.abs(lng) <= 180;
258
+ };
259
+
260
+ export const DATE_FORMS = 'a YYYY, YYYY-MM, YYYY-MM-DD or RFC 3339 date';
261
+
262
+ /**
263
+ * Check a value against a field type, with a date as its first UTC instant
264
+ */
265
+ export const storeScalar = (type: FieldType, value: unknown): { value: Scalar | GeoPoint } | { problem: string } => {
266
+ switch (type) {
267
+ case 'text':
268
+ case 'keyword':
269
+ return typeof value === 'string' ? { value } : { problem: 'must be a string' };
270
+ case 'date': {
271
+ const date = typeof value === 'string' ? parseDate(value) : undefined;
272
+ return date ? { value: date } : { problem: `must be ${DATE_FORMS}` };
273
+ }
274
+ case 'number':
275
+ return typeof value === 'number' && Number.isFinite(value) ? { value } : { problem: 'must be a finite number' };
276
+ case 'boolean':
277
+ return typeof value === 'boolean' ? { value } : { problem: 'must be a boolean' };
278
+ case 'geo':
279
+ return isGeoPoint(value)
280
+ ? { value }
281
+ : { problem: 'must be { lat, lng } with lat from -90 to 90 and lng from -180 to 180' };
282
+ }
283
+ };
284
+
285
+ /**
286
+ * Validate a batch of search documents against the field declarations, throwing on the first bad one
287
+ */
288
+ export const storeDocuments = (fields: FieldDeclarations, docs: SearchDocument[]): StoredSearchDocument[] =>
289
+ docs.map(({ id, ...rest }) => {
290
+ if (typeof id !== 'string' || id === '') {
291
+ throw new Error('Search document has no id');
292
+ }
293
+
294
+ const fail = (key: string, problem: string) => {
295
+ throw new Error(`Search document "${id}": "${key}" ${problem}`);
296
+ };
297
+
298
+ const stored: StoredSearchDocument = { id, values: {} };
299
+ for (const [key, value] of Object.entries(rest)) {
300
+ const declaration = Object.hasOwn(fields, key) ? fields[key] : undefined;
301
+ if (!declaration) {
302
+ return fail(key, "isn't a declared field");
303
+ }
304
+ if (Array.isArray(value) && (declaration.sort || declaration.type === 'geo')) {
305
+ return fail(key, 'takes a single value, not an array');
306
+ }
307
+
308
+ const items = (Array.isArray(value) ? value : [value]).map((item) => {
309
+ const result = storeScalar(declaration.type, item);
310
+ return 'value' in result ? result.value : fail(key, `${result.problem}, not ${JSON.stringify(item)}`);
311
+ });
312
+ if (declaration.type === 'geo') {
313
+ stored.point = items[0] as GeoPoint;
314
+ } else {
315
+ stored.values[key] = Array.isArray(value) ? (items as Scalar[]) : (items[0] as Scalar);
316
+ }
317
+ }
318
+
319
+ return stored;
320
+ });
@@ -0,0 +1,36 @@
1
+ import type { QueryContainer } from '@opensearch-project/opensearch/api/_types/_common.query_dsl.js';
2
+ import { matchedDateSpan } from '../advanced.js';
3
+ import type { AdvancedQuery } from '../backend.js';
4
+ import type { FieldDeclarations } from '../fields.js';
5
+
6
+ /**
7
+ * Translate an advanced query tree to OpenSearch DSL: text matches all words or the phrase, other types match exactly
8
+ */
9
+ export const advancedQueryFor = (query: AdvancedQuery, fields: FieldDeclarations): QueryContainer => {
10
+ switch (query.type) {
11
+ case 'and':
12
+ return { bool: { must: query.clauses.map((clause) => advancedQueryFor(clause, fields)) } };
13
+ case 'or':
14
+ return {
15
+ bool: { should: query.clauses.map((clause) => advancedQueryFor(clause, fields)), minimum_should_match: 1 },
16
+ };
17
+ case 'not':
18
+ return { bool: { must_not: [advancedQueryFor(query.clause, fields)] } };
19
+ case 'match': {
20
+ const { field, value, phrase } = query;
21
+ const { type } = fields[field];
22
+ if (type === 'date') {
23
+ const { first, last } = matchedDateSpan(field, value);
24
+
25
+ return { range: { [field]: { gte: first.toISOString(), lte: last.toISOString() } } };
26
+ }
27
+ if (type !== 'text') {
28
+ return { term: { [field]: value } };
29
+ }
30
+
31
+ return phrase ? { match_phrase: { [field]: value } } : { match: { [field]: { query: value, operator: 'and' } } };
32
+ }
33
+ default:
34
+ return { match_all: {} };
35
+ }
36
+ };
@@ -0,0 +1,290 @@
1
+ import { isDeepStrictEqual } from 'node:util';
2
+ import type { Client } from '@opensearch-project/opensearch';
3
+ import type { SortOrder } from '@opensearch-project/opensearch/api/_types/_common.js';
4
+ import type { QueryContainer } from '@opensearch-project/opensearch/api/_types/_common.query_dsl.js';
5
+ import type { Search_RequestBody, Search_Response } from '@opensearch-project/opensearch/api/index.js';
6
+ import {
7
+ type BoundingBox,
8
+ type FacetBucket,
9
+ GEOHASH_GRID_CELLS,
10
+ SearchQueryError,
11
+ type SearchRequest,
12
+ type SearchResult,
13
+ } from '../backend.js';
14
+ import {
15
+ capabilitiesFromFields,
16
+ type FieldDeclaration,
17
+ type FieldDeclarations,
18
+ isBasic,
19
+ resolveFields,
20
+ type SearchDocument,
21
+ storeDocuments,
22
+ } from '../fields.js';
23
+ import { filtersCondition } from '../requestFilters.js';
24
+ import { checkSearchVisibility } from '../visibility.js';
25
+ import { advancedQueryFor } from './advanced.js';
26
+ import { visibilityQuery } from './visibility.js';
27
+
28
+ export type OpenSearchBackendOptions<Fields extends FieldDeclarations> = {
29
+ client: Client;
30
+ fields: Fields;
31
+ /** Default `entities` */
32
+ indexName?: string;
33
+ /** Passed to each bulk `index` and `remove`. Default `false` */
34
+ refresh?: boolean | 'wait_for';
35
+ };
36
+
37
+ // A keyword term is limited to 32766 bytes, up to 4 per character
38
+ const EXACT_FORM = { exact: { type: 'keyword', ignore_above: 8191 } };
39
+
40
+ const mappingFor = (declaration: FieldDeclaration) => {
41
+ const { type } = declaration;
42
+ switch (type) {
43
+ case 'text':
44
+ return { type, fields: EXACT_FORM };
45
+ case 'number':
46
+ return { type: 'double' };
47
+ case 'geo':
48
+ return { type: 'geo_point' };
49
+ default:
50
+ return { type };
51
+ }
52
+ };
53
+
54
+ const FACET_ORDER: Record<string, SortOrder>[] = [{ _count: 'desc' }, { _key: 'asc' }];
55
+
56
+ type TermsBucket = { key: string | number; key_as_string?: string; doc_count: number };
57
+
58
+ // Not a valid field name, so it can't collide with a facet
59
+ const GEOHASH_GRID = 'geohash-grid';
60
+
61
+ const geoBox = ({ topRight, bottomLeft }: BoundingBox) => ({
62
+ top_left: { lat: topRight.lat, lon: bottomLeft.lng },
63
+ bottom_right: { lat: bottomLeft.lat, lon: topRight.lng },
64
+ });
65
+
66
+ type BulkItem = Record<
67
+ string,
68
+ { _id?: string | null; status: number; error?: { type: string; reason?: string | null } }
69
+ >;
70
+
71
+ /**
72
+ * OpenSearch search backend: holds one search document per entity in an index built from the field declarations
73
+ */
74
+ export const OpenSearchBackend = <const Fields extends FieldDeclarations>({
75
+ client,
76
+ fields,
77
+ indexName = 'entities',
78
+ refresh = false,
79
+ }: OpenSearchBackendOptions<Fields>) => {
80
+ const capabilities = capabilitiesFromFields(fields);
81
+ const { basic, facets, geoField } = resolveFields(fields);
82
+
83
+ const exactField = (name: string) => (fields[name]?.type === 'text' ? `${name}.exact` : name);
84
+ const basicFields = basic.map(({ name, weight }) => `${name}^${weight}`);
85
+ const highlight: Search_RequestBody['highlight'] = {
86
+ fields: Object.fromEntries(
87
+ Object.entries(fields)
88
+ .filter(
89
+ ([, declaration]) =>
90
+ ['text', 'keyword'].includes(declaration.type) && (isBasic(declaration) || declaration.advanced),
91
+ )
92
+ .map(([name]) => [name, {}]),
93
+ ),
94
+ };
95
+
96
+ const queryFor = (request: SearchRequest): QueryContainer => {
97
+ if (request.searchType === 'advanced') {
98
+ return advancedQueryFor(request.query, fields);
99
+ }
100
+ if (request.query.trim() === '') {
101
+ return { match_all: {} };
102
+ }
103
+ if (basicFields.length === 0) {
104
+ return { match_none: {} };
105
+ }
106
+
107
+ return {
108
+ multi_match: {
109
+ query: request.query,
110
+ fields: basicFields,
111
+ type: 'best_fields',
112
+ fuzziness: 'AUTO',
113
+ zero_terms_query: 'all',
114
+ },
115
+ };
116
+ };
117
+
118
+ const sortFor = ({ sort, order }: SearchRequest): Search_RequestBody['sort'] => [
119
+ sort === 'relevance' ? { _score: { order: 'desc' } } : { [exactField(sort)]: { order, missing: '_last' } },
120
+ { id: { order: 'asc' } },
121
+ ];
122
+
123
+ const aggregations: Search_RequestBody['aggs'] = Object.fromEntries(
124
+ facets.map(({ name, type, size }) => [
125
+ name,
126
+ type === 'date'
127
+ ? {
128
+ date_histogram: {
129
+ field: name,
130
+ calendar_interval: 'year',
131
+ format: 'yyyy',
132
+ time_zone: 'UTC',
133
+ min_doc_count: 1,
134
+ order: { _key: 'asc' },
135
+ },
136
+ }
137
+ : {
138
+ terms: {
139
+ field: exactField(name),
140
+ size,
141
+ order: FACET_ORDER,
142
+ },
143
+ },
144
+ ]),
145
+ );
146
+
147
+ const properties: Record<string, object> = {
148
+ id: { type: 'keyword' },
149
+ ...Object.fromEntries(Object.entries(fields).map(([name, declaration]) => [name, mappingFor(declaration)])),
150
+ };
151
+
152
+ const bulk = async (operations: object[], tolerated: number[] = []) => {
153
+ if (operations.length === 0) {
154
+ return;
155
+ }
156
+
157
+ const { body } = await client.bulk({ body: operations, refresh });
158
+ const failure = (body.items as BulkItem[])
159
+ .map((item) => Object.values(item)[0])
160
+ .find(({ error, status }) => error && !tolerated.includes(status));
161
+ if (failure?.error) {
162
+ const { _id, error } = failure;
163
+ throw new Error(`Search document "${_id}" failed in index "${indexName}": ${error.type}: ${error.reason}`);
164
+ }
165
+ };
166
+
167
+ return {
168
+ capabilities,
169
+
170
+ search: async (request: SearchRequest): Promise<SearchResult> => {
171
+ const { boundingBox, geohashPrecision } = request;
172
+ const filter: QueryContainer[] = [
173
+ visibilityQuery(filtersCondition(request.filters), exactField),
174
+ ...(geoField && boundingBox ? [{ geo_bounding_box: { [geoField]: geoBox(boundingBox) } }] : []),
175
+ visibilityQuery(checkSearchVisibility(fields, request.visibility), exactField),
176
+ ];
177
+ const geohashGrid: Search_RequestBody['aggs'] =
178
+ geoField && geohashPrecision !== undefined
179
+ ? {
180
+ [GEOHASH_GRID]: {
181
+ geohash_grid: { field: geoField, precision: geohashPrecision, size: GEOHASH_GRID_CELLS },
182
+ },
183
+ }
184
+ : {};
185
+
186
+ let response: Search_Response;
187
+ try {
188
+ response = await client.search({
189
+ index: indexName,
190
+ body: {
191
+ query: { bool: { must: [queryFor(request)], filter } },
192
+ aggs: { ...aggregations, ...geohashGrid },
193
+ highlight,
194
+ sort: sortFor(request),
195
+ from: request.offset,
196
+ size: request.limit,
197
+ track_total_hits: true,
198
+ _source: false,
199
+ },
200
+ });
201
+ } catch (error) {
202
+ const { statusCode, message } = error as Error & { statusCode?: number };
203
+ if (statusCode === 400) {
204
+ throw new SearchQueryError(message);
205
+ }
206
+
207
+ throw error;
208
+ }
209
+
210
+ const { hits, aggregations: buckets = {} } = response.body;
211
+
212
+ return {
213
+ total: (hits.total as { value: number }).value,
214
+ hits: hits.hits.map(({ _id, _score, highlight: highlighted }) => {
215
+ const searchExtra = {
216
+ ...(typeof _score === 'number' && { score: _score }),
217
+ ...(highlighted && Object.keys(highlighted).length > 0 && { highlight: highlighted }),
218
+ };
219
+
220
+ return { id: _id as string, ...(Object.keys(searchExtra).length > 0 && { searchExtra }) };
221
+ }),
222
+ facets: Object.fromEntries(
223
+ facets.map(({ name }) => [
224
+ name,
225
+ (buckets[name] as { buckets: TermsBucket[] }).buckets.map(
226
+ ({ key, key_as_string, doc_count }): FacetBucket => ({
227
+ name: key_as_string ?? String(key),
228
+ count: doc_count,
229
+ }),
230
+ ),
231
+ ]),
232
+ ),
233
+ ...(GEOHASH_GRID in buckets && {
234
+ geohashGrid: Object.fromEntries(
235
+ (buckets[GEOHASH_GRID] as { buckets: TermsBucket[] }).buckets.map(({ key, doc_count }) => [key, doc_count]),
236
+ ),
237
+ }),
238
+ };
239
+ },
240
+
241
+ /**
242
+ * Create the index with a mapping built from the field declarations
243
+ * Does nothing if it exists with that mapping, and throws, naming the field, if it exists with another
244
+ */
245
+ createIndex: async () => {
246
+ const { body: exists } = await client.indices.exists({ index: indexName });
247
+ if (!exists) {
248
+ await client.indices.create({ index: indexName, body: { mappings: { dynamic: 'strict', properties } } });
249
+
250
+ return;
251
+ }
252
+
253
+ const { body } = await client.indices.getMapping({ index: indexName });
254
+ const existing = (body[indexName].mappings.properties ?? {}) as Record<string, unknown>;
255
+ const differing = [...new Set([...Object.keys(properties), ...Object.keys(existing)])].find(
256
+ (name) => !isDeepStrictEqual(properties[name], existing[name]),
257
+ );
258
+ if (differing) {
259
+ throw new Error(
260
+ `Index "${indexName}" exists with a different mapping for field "${differing}" than its declaration: ` +
261
+ `${JSON.stringify(existing[differing] ?? null)} instead of ${JSON.stringify(properties[differing] ?? null)}`,
262
+ );
263
+ }
264
+ },
265
+
266
+ /**
267
+ * Upsert search documents, replacing any already indexed with the same id
268
+ */
269
+ index: async (docs: SearchDocument<Fields>[]) => {
270
+ const stored = storeDocuments(fields, docs);
271
+
272
+ await bulk(
273
+ stored.flatMap(({ id, values, point }) => [
274
+ { index: { _index: indexName, _id: id } },
275
+ { id, ...values, ...(geoField && point && { [geoField]: { lat: point.lat, lon: point.lng } }) },
276
+ ]),
277
+ );
278
+ },
279
+
280
+ /**
281
+ * Remove search documents, ignoring ids that aren't indexed
282
+ */
283
+ remove: async (ids: string[]) => {
284
+ await bulk(
285
+ ids.map((id) => ({ delete: { _index: indexName, _id: id } })),
286
+ [404],
287
+ );
288
+ },
289
+ };
290
+ };
@@ -0,0 +1,38 @@
1
+ import type { QueryContainer } from '@opensearch-project/opensearch/api/_types/_common.query_dsl.js';
2
+ import { toWire } from '../fields.js';
3
+ import type { VisibilityCondition } from '../visibility.js';
4
+
5
+ /**
6
+ * Translate a checked search visibility filter to an OpenSearch filter, comparing each field's exact form
7
+ */
8
+ export const visibilityQuery = (
9
+ condition: VisibilityCondition,
10
+ exactField: (name: string) => string,
11
+ ): QueryContainer => {
12
+ const translate = (node: VisibilityCondition): QueryContainer => {
13
+ switch (node.type) {
14
+ case 'and':
15
+ return { bool: { filter: node.clauses.map(translate) } };
16
+ case 'or':
17
+ return node.clauses.length === 0
18
+ ? { match_none: {} }
19
+ : { bool: { should: node.clauses.map(translate), minimum_should_match: 1 } };
20
+ case 'not':
21
+ return { bool: { must_not: [translate(node.clause)] } };
22
+ case 'in':
23
+ return { terms: { [exactField(node.field)]: node.values.map(toWire) } };
24
+ case 'present':
25
+ return { exists: { field: node.field } };
26
+ case 'range':
27
+ return {
28
+ range: {
29
+ [node.field]: Object.fromEntries(
30
+ Object.entries(node.bounds).map(([bound, value]) => [bound, toWire(value)]),
31
+ ),
32
+ },
33
+ };
34
+ }
35
+ };
36
+
37
+ return translate(condition);
38
+ };