arocapi 5.0.0-beta.1 → 5.0.0-beta.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (234) hide show
  1. package/LICENSE.md +1 -1
  2. package/README.md +37 -802
  3. package/dist/app.d.ts +38 -20
  4. package/dist/app.js +62 -40
  5. package/dist/app.js.map +1 -1
  6. package/dist/routes/capabilities.d.ts +7 -0
  7. package/dist/routes/capabilities.js +13 -0
  8. package/dist/routes/capabilities.js.map +1 -0
  9. package/dist/routes/entities.d.ts +5 -2
  10. package/dist/routes/entities.js +37 -57
  11. package/dist/routes/entities.js.map +1 -1
  12. package/dist/routes/entity.d.ts +7 -2
  13. package/dist/routes/entity.js +13 -29
  14. package/dist/routes/entity.js.map +1 -1
  15. package/dist/routes/file.d.ts +8 -2
  16. package/dist/routes/file.js +37 -72
  17. package/dist/routes/file.js.map +1 -1
  18. package/dist/routes/files.d.ts +5 -2
  19. package/dist/routes/files.js +24 -39
  20. package/dist/routes/files.js.map +1 -1
  21. package/dist/routes/metadata.d.ts +11 -5
  22. package/dist/routes/metadata.js +24 -68
  23. package/dist/routes/metadata.js.map +1 -1
  24. package/dist/routes/search.d.ts +8 -7
  25. package/dist/routes/search.js +90 -119
  26. package/dist/routes/search.js.map +1 -1
  27. package/dist/search/advanced.d.ts +8 -0
  28. package/dist/search/advanced.js +198 -0
  29. package/dist/search/advanced.js.map +1 -0
  30. package/dist/search/backend.d.ts +95 -0
  31. package/dist/search/backend.js +5 -0
  32. package/dist/search/backend.js.map +1 -0
  33. package/dist/search/fields.d.ts +63 -0
  34. package/dist/search/fields.js +203 -0
  35. package/dist/search/fields.js.map +1 -0
  36. package/dist/search/opensearch/advanced.d.ts +4 -0
  37. package/dist/search/opensearch/advanced.js +28 -0
  38. package/dist/search/opensearch/advanced.js.map +1 -0
  39. package/dist/search/opensearch/opensearch.d.ts +16 -0
  40. package/dist/search/opensearch/opensearch.js +187 -0
  41. package/dist/search/opensearch/opensearch.js.map +1 -0
  42. package/dist/search/opensearch/visibility.d.ts +3 -0
  43. package/dist/search/opensearch/visibility.js +27 -0
  44. package/dist/search/opensearch/visibility.js.map +1 -0
  45. package/dist/search/postgres/advanced.d.ts +7 -0
  46. package/dist/search/postgres/advanced.js +54 -0
  47. package/dist/search/postgres/advanced.js.map +1 -0
  48. package/dist/search/postgres/facets.d.ts +2 -0
  49. package/dist/search/postgres/facets.js +27 -0
  50. package/dist/search/postgres/facets.js.map +1 -0
  51. package/dist/search/postgres/geo.d.ts +6 -0
  52. package/dist/search/postgres/geo.js +42 -0
  53. package/dist/search/postgres/geo.js.map +1 -0
  54. package/dist/search/postgres/postgres.d.ts +19 -0
  55. package/dist/search/postgres/postgres.js +157 -0
  56. package/dist/search/postgres/postgres.js.map +1 -0
  57. package/dist/search/postgres/values.d.ts +8 -0
  58. package/dist/search/postgres/values.js +20 -0
  59. package/dist/search/postgres/values.js.map +1 -0
  60. package/dist/search/postgres/visibility.d.ts +4 -0
  61. package/dist/search/postgres/visibility.js +38 -0
  62. package/dist/search/postgres/visibility.js.map +1 -0
  63. package/dist/search/requestFilters.d.ts +39 -0
  64. package/dist/search/requestFilters.js +125 -0
  65. package/dist/search/requestFilters.js.map +1 -0
  66. package/dist/search/visibility.d.ts +51 -0
  67. package/dist/search/visibility.js +100 -0
  68. package/dist/search/visibility.js.map +1 -0
  69. package/dist/storage/backend.d.ts +58 -0
  70. package/dist/storage/backend.js +2 -0
  71. package/dist/storage/backend.js.map +1 -0
  72. package/dist/storage/deliver.d.ts +12 -0
  73. package/dist/storage/deliver.js +34 -0
  74. package/dist/storage/deliver.js.map +1 -0
  75. package/dist/storage/directory/directory.d.ts +6 -0
  76. package/dist/storage/directory/directory.js +24 -0
  77. package/dist/storage/directory/directory.js.map +1 -0
  78. package/dist/storage/entity-meta/entity-meta.d.ts +5 -0
  79. package/dist/storage/entity-meta/entity-meta.js +14 -0
  80. package/dist/storage/entity-meta/entity-meta.js.map +1 -0
  81. package/dist/storage/location.d.ts +19 -0
  82. package/dist/storage/location.js +36 -0
  83. package/dist/storage/location.js.map +1 -0
  84. package/dist/storage/ocfl/ocfl.d.ts +6 -0
  85. package/dist/storage/ocfl/ocfl.js +45 -0
  86. package/dist/storage/ocfl/ocfl.js.map +1 -0
  87. package/dist/storage/results.d.ts +7 -0
  88. package/dist/storage/results.js +22 -0
  89. package/dist/storage/results.js.map +1 -0
  90. package/dist/storage/s3/s3.d.ts +10 -0
  91. package/dist/storage/s3/s3.js +47 -0
  92. package/dist/storage/s3/s3.js.map +1 -0
  93. package/dist/transformers/default.d.ts +11 -22
  94. package/dist/transformers/default.js +41 -15
  95. package/dist/transformers/default.js.map +1 -1
  96. package/dist/types/access.d.ts +9 -0
  97. package/dist/types/access.js +2 -0
  98. package/dist/types/access.js.map +1 -0
  99. package/dist/types/context.d.ts +6 -0
  100. package/dist/types/context.js +2 -0
  101. package/dist/types/context.js.map +1 -0
  102. package/dist/types/transformers.d.ts +4 -10
  103. package/dist/types/visibility.d.ts +6 -0
  104. package/dist/types/visibility.js +2 -0
  105. package/dist/types/visibility.js.map +1 -0
  106. package/dist/utils/access.d.ts +17 -0
  107. package/dist/utils/access.js +41 -0
  108. package/dist/utils/access.js.map +1 -0
  109. package/dist/utils/errors.d.ts +19 -10
  110. package/dist/utils/errors.js +44 -8
  111. package/dist/utils/errors.js.map +1 -1
  112. package/dist/utils/headers.d.ts +1 -1
  113. package/dist/utils/headers.js +10 -1
  114. package/dist/utils/headers.js.map +1 -1
  115. package/dist/utils/validation.d.ts +2 -0
  116. package/dist/utils/validation.js +8 -0
  117. package/dist/utils/validation.js.map +1 -0
  118. package/dist/utils/visibility.d.ts +18 -0
  119. package/dist/utils/visibility.js +16 -0
  120. package/dist/utils/visibility.js.map +1 -0
  121. package/package.json +82 -40
  122. package/prisma/models/entity.prisma +9 -8
  123. package/prisma/models/entitySearch.prisma +21 -0
  124. package/prisma/models/file.prisma +3 -3
  125. package/src/app.ts +168 -95
  126. package/src/routes/capabilities.ts +22 -0
  127. package/src/routes/entities.ts +48 -67
  128. package/src/routes/entity.ts +21 -35
  129. package/src/routes/file.ts +53 -103
  130. package/src/routes/files.ts +31 -47
  131. package/src/routes/metadata.ts +42 -98
  132. package/src/routes/search.ts +131 -151
  133. package/src/search/advanced.ts +258 -0
  134. package/src/search/backend.ts +91 -0
  135. package/src/search/fields.ts +320 -0
  136. package/src/search/opensearch/advanced.ts +36 -0
  137. package/src/search/opensearch/opensearch.ts +290 -0
  138. package/src/search/opensearch/visibility.ts +38 -0
  139. package/src/search/postgres/advanced.ts +76 -0
  140. package/src/search/postgres/facets.ts +34 -0
  141. package/src/search/postgres/geo.ts +58 -0
  142. package/src/search/postgres/postgres.ts +232 -0
  143. package/src/search/postgres/values.ts +33 -0
  144. package/src/search/postgres/visibility.ts +54 -0
  145. package/src/search/requestFilters.ts +178 -0
  146. package/src/search/visibility.ts +193 -0
  147. package/src/storage/backend.ts +75 -0
  148. package/src/storage/deliver.ts +65 -0
  149. package/src/storage/directory/directory.ts +39 -0
  150. package/src/storage/entity-meta/entity-meta.ts +25 -0
  151. package/src/storage/location.ts +78 -0
  152. package/src/storage/ocfl/ocfl.ts +71 -0
  153. package/src/storage/results.ts +32 -0
  154. package/src/storage/s3/s3.ts +79 -0
  155. package/src/transformers/default.ts +115 -85
  156. package/src/types/access.ts +25 -0
  157. package/src/types/context.ts +15 -0
  158. package/src/types/transformers.ts +8 -34
  159. package/src/types/visibility.ts +21 -0
  160. package/src/utils/access.ts +80 -0
  161. package/src/utils/errors.ts +68 -11
  162. package/src/utils/headers.ts +22 -2
  163. package/src/utils/validation.ts +11 -0
  164. package/src/utils/visibility.ts +56 -0
  165. package/dist/app.test.d.ts +0 -1
  166. package/dist/app.test.js +0 -82
  167. package/dist/app.test.js.map +0 -1
  168. package/dist/index.dev.d.ts +0 -1
  169. package/dist/index.dev.js +0 -156
  170. package/dist/index.dev.js.map +0 -1
  171. package/dist/routes/crate.test.d.ts +0 -1
  172. package/dist/routes/crate.test.js +0 -342
  173. package/dist/routes/crate.test.js.map +0 -1
  174. package/dist/routes/entities.test.d.ts +0 -1
  175. package/dist/routes/entities.test.js +0 -273
  176. package/dist/routes/entities.test.js.map +0 -1
  177. package/dist/routes/entity.test.d.ts +0 -1
  178. package/dist/routes/entity.test.js +0 -130
  179. package/dist/routes/entity.test.js.map +0 -1
  180. package/dist/routes/file.test.d.ts +0 -1
  181. package/dist/routes/file.test.js +0 -315
  182. package/dist/routes/file.test.js.map +0 -1
  183. package/dist/routes/files.test.d.ts +0 -1
  184. package/dist/routes/files.test.js +0 -222
  185. package/dist/routes/files.test.js.map +0 -1
  186. package/dist/routes/search.test.d.ts +0 -1
  187. package/dist/routes/search.test.js +0 -780
  188. package/dist/routes/search.test.js.map +0 -1
  189. package/dist/test/helpers/fastify.d.ts +0 -9
  190. package/dist/test/helpers/fastify.js +0 -18
  191. package/dist/test/helpers/fastify.js.map +0 -1
  192. package/dist/test/integration.setup.d.ts +0 -29
  193. package/dist/test/integration.setup.js +0 -187
  194. package/dist/test/integration.setup.js.map +0 -1
  195. package/dist/test/integration.test.d.ts +0 -1
  196. package/dist/test/integration.test.js +0 -396
  197. package/dist/test/integration.test.js.map +0 -1
  198. package/dist/transformers/default.test.d.ts +0 -1
  199. package/dist/transformers/default.test.js +0 -163
  200. package/dist/transformers/default.test.js.map +0 -1
  201. package/dist/transformers/transformer.test.d.ts +0 -1
  202. package/dist/transformers/transformer.test.js +0 -187
  203. package/dist/transformers/transformer.test.js.map +0 -1
  204. package/dist/types/fileHandlers.d.ts +0 -41
  205. package/dist/types/fileHandlers.js +0 -2
  206. package/dist/types/fileHandlers.js.map +0 -1
  207. package/dist/utils/errors.test.d.ts +0 -1
  208. package/dist/utils/errors.test.js +0 -66
  209. package/dist/utils/errors.test.js.map +0 -1
  210. package/dist/utils/queryBuilder.d.ts +0 -35
  211. package/dist/utils/queryBuilder.js +0 -98
  212. package/dist/utils/queryBuilder.js.map +0 -1
  213. package/prisma.config.ts +0 -9
  214. package/src/app.test.ts +0 -112
  215. package/src/index.dev.ts +0 -200
  216. package/src/reset.d.ts +0 -1
  217. package/src/routes/__snapshots__/entities.test.ts.snap +0 -96
  218. package/src/routes/__snapshots__/entity.test.ts.snap +0 -74
  219. package/src/routes/__snapshots__/search.test.ts.snap +0 -165
  220. package/src/routes/crate.test.ts +0 -423
  221. package/src/routes/entities.test.ts +0 -320
  222. package/src/routes/entity.test.ts +0 -160
  223. package/src/routes/file.test.ts +0 -395
  224. package/src/routes/files.test.ts +0 -278
  225. package/src/routes/search.test.ts +0 -903
  226. package/src/test/__snapshots__/integration.test.ts.snap +0 -18
  227. package/src/test/helpers/fastify.ts +0 -27
  228. package/src/test/integration.setup.ts +0 -209
  229. package/src/test/integration.test.ts +0 -482
  230. package/src/transformers/default.test.ts +0 -184
  231. package/src/transformers/transformer.test.ts +0 -252
  232. package/src/types/fileHandlers.ts +0 -65
  233. package/src/utils/errors.test.ts +0 -75
  234. package/src/utils/queryBuilder.ts +0 -132
@@ -0,0 +1,232 @@
1
+ import { join, type Sql, sqltag as sql } from '@prisma/client/runtime/client';
2
+ import { SearchQueryError, type SearchRequest, type SearchResult } from '../backend.js';
3
+ import {
4
+ capabilitiesFromFields,
5
+ type FieldDeclarations,
6
+ resolveFields,
7
+ type SearchDocument,
8
+ storeDocuments,
9
+ toWire,
10
+ } from '../fields.js';
11
+ import { filtersCondition } from '../requestFilters.js';
12
+ import { checkSearchVisibility } from '../visibility.js';
13
+ import { advancedQueryFor } from './advanced.js';
14
+ import { valueBuckets, yearBuckets } from './facets.js';
15
+ import { boundingBoxCondition, geohash, geohashGrid } from './geo.js';
16
+ import { TEXT_SEARCH_CONFIG, typed } from './values.js';
17
+ import { visibilityCondition } from './visibility.js';
18
+
19
+ /**
20
+ * The Prisma client PostgresBackend needs: any Prisma 7 client on a schema that includes the arocapi models,
21
+ * including the client inside `$transaction`
22
+ */
23
+ export type PostgresClient = {
24
+ $queryRaw<T = unknown>(query: Sql): PromiseLike<T>;
25
+ $executeRaw(query: Sql): PromiseLike<number>;
26
+ };
27
+
28
+ export type PostgresBackendOptions<Fields extends FieldDeclarations> = {
29
+ prisma: PostgresClient;
30
+ fields: Fields;
31
+ };
32
+
33
+ const WEIGHT_LABELS = ['D', 'C', 'B', 'A'];
34
+
35
+ /**
36
+ * Postgres search backend: holds one search document per entity in the `EntitySearch` table
37
+ */
38
+ export const PostgresBackend = <const Fields extends FieldDeclarations>({
39
+ prisma,
40
+ fields,
41
+ }: PostgresBackendOptions<Fields>) => {
42
+ const capabilities = capabilitiesFromFields(fields);
43
+ const { basic, facets } = resolveFields(fields);
44
+ const textFields = Object.keys(fields).filter((name) => fields[name].type === 'text');
45
+
46
+ const basicVectors = basic.map(
47
+ ({ name, weight }) =>
48
+ sql`coalesce(setweight(to_tsvector(${TEXT_SEARCH_CONFIG}, docs."values" -> ${name}), ${WEIGHT_LABELS[weight - 1]}::"char"), '')`,
49
+ );
50
+ const basicVector = basicVectors.length > 0 ? join(basicVectors, ' || ') : sql`''::tsvector`;
51
+
52
+ // Built as text so each field keeps its own positions; tsvector `||` would shift them towards the 16383 limit
53
+ const textVector = sql`(
54
+ SELECT coalesce(string_agg(
55
+ '''' || replace(replace(field || '.' || lexeme, '\\', '\\\\'), '''', '''''') || ''':' || array_to_string(positions, ','),
56
+ ' '
57
+ ), '')::tsvector
58
+ FROM unnest(${textFields}::text[]) AS field, unnest(to_tsvector(${TEXT_SEARCH_CONFIG}, docs."values" -> field))
59
+ )`;
60
+
61
+ const facetBuckets = facets.map(
62
+ ({ name, type, size }) =>
63
+ sql`jsonb_build_object(${name}::text, ${type === 'date' ? yearBuckets(name) : valueBuckets(name, size)})`,
64
+ );
65
+
66
+ const queryFor = (request: SearchRequest) => {
67
+ if (request.searchType === 'advanced') {
68
+ return advancedQueryFor(request.query, fields);
69
+ }
70
+ if (request.query.trim() === '') {
71
+ return { condition: sql`TRUE`, rank: sql`0` };
72
+ }
73
+ const tsquery = sql`websearch_to_tsquery(${TEXT_SEARCH_CONFIG}, ${request.query}::text)`;
74
+
75
+ // A query with no words left after parsing, such as `*`, matches everything
76
+ return {
77
+ condition: sql`(numnode(${tsquery}) = 0 OR search.basic @@ ${tsquery})`,
78
+ rank: sql`ts_rank(search.basic, ${tsquery})`,
79
+ };
80
+ };
81
+
82
+ return {
83
+ capabilities,
84
+
85
+ search: async (request: SearchRequest): Promise<SearchResult> => {
86
+ const { condition: matches, rank } = queryFor(request);
87
+ const conditions = [
88
+ matches,
89
+ visibilityCondition(filtersCondition(request.filters), fields),
90
+ ...(request.boundingBox ? [boundingBoxCondition(request.boundingBox)] : []),
91
+ visibilityCondition(checkSearchVisibility(fields, request.visibility), fields),
92
+ ];
93
+
94
+ const { sort, order, geohashPrecision } = request;
95
+ const [key, direction] =
96
+ sort === 'relevance'
97
+ ? [rank, sql`DESC`]
98
+ : [
99
+ typed(fields[sort].type, sql`search."values" ->> ${sort}::text`),
100
+ order === 'asc' ? sql`ASC NULLS LAST` : sql`DESC NULLS LAST`,
101
+ ];
102
+ const [result] = await asQueryErrors(
103
+ prisma.$queryRaw<(Pick<SearchResult, 'total' | 'facets' | 'geohashGrid'> & { hits: string[] })[]>(sql`
104
+ WITH matched AS (
105
+ SELECT search.id, search."values", search.geohash, ${key} AS key
106
+ FROM entity_search AS search
107
+ WHERE ${join(conditions, ' AND ')}
108
+ )
109
+ SELECT
110
+ (SELECT count(*)::int FROM matched) AS total,
111
+ (
112
+ SELECT coalesce(jsonb_agg(page.id ORDER BY page.key ${direction}, page.id COLLATE "C"), '[]')
113
+ FROM (
114
+ SELECT search.id, search.key
115
+ FROM matched AS search
116
+ ORDER BY search.key ${direction}, search.id COLLATE "C"
117
+ LIMIT ${request.limit}::int OFFSET ${request.offset}::int
118
+ ) AS page
119
+ ) AS hits,
120
+ ${join([sql`'{}'::jsonb`, ...facetBuckets], ' || ')} AS facets,
121
+ ${geohashPrecision === undefined ? sql`NULL` : geohashGrid(geohashPrecision)} AS "geohashGrid"
122
+ `),
123
+ );
124
+
125
+ return {
126
+ total: result.total,
127
+ hits: result.hits.map((id) => ({ id })),
128
+ facets: result.facets,
129
+ ...(result.geohashGrid && { geohashGrid: result.geohashGrid }),
130
+ };
131
+ },
132
+
133
+ /**
134
+ * Upsert search documents, replacing any already indexed with the same id
135
+ * Throws, naming them, if any entities don't exist, and then stores nothing from the batch
136
+ */
137
+ index: async (docs: SearchDocument<Fields>[], { tx }: { tx?: PostgresClient } = {}) => {
138
+ const stored = storeDocuments(fields, docs);
139
+
140
+ const rows = [...new Map(stored.map((doc) => [doc.id, doc])).values()].map(({ id, values, point }) => ({
141
+ id,
142
+ values: Object.fromEntries(
143
+ Object.entries(values).map(([name, value]) => [
144
+ name,
145
+ Array.isArray(value) ? value.map(toWire) : toWire(value),
146
+ ]),
147
+ ),
148
+ ...(point && { ...point, geohash: geohash(point) }),
149
+ }));
150
+
151
+ const ids = rows.map(({ id }) => id);
152
+ const upsert = (tx ?? prisma).$queryRaw<{ id: string }[]>(sql`
153
+ WITH docs AS (
154
+ SELECT * FROM jsonb_to_recordset(${JSON.stringify(rows)}::jsonb)
155
+ AS docs(id text, "values" jsonb, lat float8, lng float8, geohash text)
156
+ ),
157
+ missing AS (
158
+ SELECT docs.id FROM docs WHERE NOT EXISTS (SELECT FROM entity WHERE entity.id = docs.id)
159
+ ),
160
+ upserted AS (
161
+ INSERT INTO entity_search (id, "values", basic, text, point, geohash)
162
+ SELECT docs.id, docs."values", ${basicVector}, ${textVector}, point(docs.lng, docs.lat), docs.geohash
163
+ FROM docs
164
+ WHERE NOT EXISTS (SELECT FROM missing)
165
+ ON CONFLICT (id) DO UPDATE SET
166
+ "values" = excluded."values",
167
+ basic = excluded.basic,
168
+ text = excluded.text,
169
+ point = excluded.point,
170
+ geohash = excluded.geohash
171
+ )
172
+ SELECT id FROM missing ORDER BY id
173
+ `);
174
+
175
+ let missing: { id: string }[];
176
+ try {
177
+ missing = await upsert;
178
+ } catch (error) {
179
+ if (driverErrorOf(error)?.originalCode !== FOREIGN_KEY_VIOLATION) {
180
+ throw error;
181
+ }
182
+
183
+ // An entity was deleted after the check; `tx` is aborted, so look outside it
184
+ missing = await prisma.$queryRaw<{ id: string }[]>(sql`
185
+ SELECT doc.id FROM unnest(${ids}::text[]) AS doc(id)
186
+ WHERE NOT EXISTS (SELECT FROM entity WHERE entity.id = doc.id)
187
+ ORDER BY doc.id
188
+ `);
189
+ if (missing.length === 0) {
190
+ throw error;
191
+ }
192
+ }
193
+
194
+ if (missing.length > 0) {
195
+ throw new Error(
196
+ `Search documents reference entities that don't exist: ${missing.map(({ id }) => JSON.stringify(id)).join(', ')}`,
197
+ );
198
+ }
199
+ },
200
+
201
+ /**
202
+ * Remove search documents, ignoring ids that aren't indexed
203
+ */
204
+ remove: async (ids: string[]) => {
205
+ await prisma.$executeRaw(sql`DELETE FROM entity_search WHERE id = ANY(${ids}::text[])`);
206
+ },
207
+ };
208
+ };
209
+
210
+ type DriverAdapterError = {
211
+ meta?: { driverAdapterError?: { cause?: { originalCode?: string; originalMessage?: string } } };
212
+ };
213
+
214
+ const driverErrorOf = (error: unknown) => (error as DriverAdapterError).meta?.driverAdapterError?.cause;
215
+
216
+ const FOREIGN_KEY_VIOLATION = '23503';
217
+
218
+ // Data exceptions (22) come from query values a field can't hold; program limits (54) from queries too big to run
219
+ const QUERY_ERROR_CLASSES = ['22', '54'];
220
+
221
+ const asQueryErrors = async <T>(query: PromiseLike<T>) => {
222
+ try {
223
+ return await query;
224
+ } catch (error) {
225
+ const cause = driverErrorOf(error);
226
+ if (cause && QUERY_ERROR_CLASSES.some((errorClass) => cause.originalCode?.startsWith(errorClass))) {
227
+ throw new SearchQueryError(cause.originalMessage);
228
+ }
229
+
230
+ throw error;
231
+ }
232
+ };
@@ -0,0 +1,33 @@
1
+ import { type Sql, sqltag as sql } from '@prisma/client/runtime/client';
2
+ import { type FieldType, type Scalar, toWire } from '../fields.js';
3
+
4
+ export const TEXT_SEARCH_CONFIG = sql`'simple'::regconfig`;
5
+
6
+ /** A JSON path to a field's values in a search row: none when missing, one per array item */
7
+ export const valuesPath = (field: string) => `$."${field}"[*]`;
8
+
9
+ /**
10
+ * A `FROM` item of a search row's values for a field, as `field.value` text
11
+ */
12
+ export const valuesOf = (field: string) =>
13
+ sql`LATERAL (SELECT jsonb_path_query(search."values", ${valuesPath(field)}::jsonpath) #>> '{}' AS value) AS field`;
14
+
15
+ /**
16
+ * Text as the field's type, comparing as OpenSearch does: strings in code point order
17
+ */
18
+ export const typed = (type: FieldType, text: Sql) => {
19
+ switch (type) {
20
+ case 'number':
21
+ return sql`(${text})::float8`;
22
+ case 'date':
23
+ return sql`(${text})::timestamptz`;
24
+ case 'boolean':
25
+ return sql`(${text})::boolean`;
26
+ default:
27
+ return sql`(${text}) COLLATE "C"`;
28
+ }
29
+ };
30
+
31
+ export const fieldValue = (type: FieldType) => typed(type, sql`field.value`);
32
+
33
+ export const parameter = (type: FieldType, value: Scalar) => typed(type, sql`${String(toWire(value))}::text`);
@@ -0,0 +1,54 @@
1
+ import { join, type Sql, sqltag as sql } from '@prisma/client/runtime/client';
2
+ import { type FieldDeclarations, type FieldType, type Scalar, toWire } from '../fields.js';
3
+ import type { VisibilityCondition } from '../visibility.js';
4
+ import { fieldValue, parameter, valuesOf, valuesPath } from './values.js';
5
+
6
+ const OPERATORS = { lt: sql`<`, lte: sql`<=`, gt: sql`>`, gte: sql`>=` };
7
+
8
+ // Boolean filter values arrive as text
9
+ const literal = (type: FieldType, value: Scalar) =>
10
+ JSON.stringify(type === 'boolean' && (value === 'true' || value === 'false') ? value === 'true' : toWire(value));
11
+
12
+ /**
13
+ * Translate a checked search visibility filter to a parameterised SQL condition on the `search` row
14
+ * Equality is JSON path containment, so the `values` index can serve it
15
+ */
16
+ export const visibilityCondition = (condition: VisibilityCondition, fields: FieldDeclarations): Sql => {
17
+ const translate = (node: VisibilityCondition): Sql => {
18
+ switch (node.type) {
19
+ case 'and':
20
+ return sql`(${join([sql`TRUE`, ...node.clauses.map(translate)], ' AND ')})`;
21
+ case 'or':
22
+ return sql`(${join([sql`FALSE`, ...node.clauses.map(translate)], ' OR ')})`;
23
+ case 'not':
24
+ return sql`NOT ${translate(node.clause)}`;
25
+ case 'present':
26
+ return node.field === 'id' ? sql`TRUE` : sql`search."values" @? ${valuesPath(node.field)}::jsonpath`;
27
+ case 'in': {
28
+ const { field, values } = node;
29
+ if (field === 'id') {
30
+ return sql`search.id = ANY(${values.map(String)}::text[])`;
31
+ }
32
+ if (values.length === 0) {
33
+ return sql`FALSE`;
34
+ }
35
+
36
+ const { type } = fields[field];
37
+ const equals = values.map((value) => `@ == ${literal(type, value)}`).join(' || ');
38
+
39
+ return sql`search."values" @? ${`${valuesPath(field)} ? (${equals})`}::jsonpath`;
40
+ }
41
+ case 'range': {
42
+ const { type } = fields[node.field];
43
+ const bounds = Object.entries(node.bounds).map(
44
+ ([bound, value]) =>
45
+ sql`${fieldValue(type)} ${OPERATORS[bound as keyof typeof OPERATORS]} ${parameter(type, value)}`,
46
+ );
47
+
48
+ return sql`EXISTS (SELECT FROM ${valuesOf(node.field)} WHERE ${join(bounds, ' AND ')})`;
49
+ }
50
+ }
51
+ };
52
+
53
+ return translate(condition);
54
+ };
@@ -0,0 +1,178 @@
1
+ import { z } from 'zod';
2
+ import type { BackendSearchCapability, BoundingBox, RangeBound, RangeFilter, SearchRequest } from './backend.js';
3
+ import { dateSpan } from './fields.js';
4
+ import type { VisibilityCondition } from './visibility.js';
5
+
6
+ const boundSchema = z.union([z.string(), z.number()]);
7
+ const rangeSchema = z
8
+ .strictObject({ gte: boundSchema.optional(), lte: boundSchema.optional() })
9
+ .refine(({ gte, lte }) => gte !== undefined || lte !== undefined, 'A range needs gte, lte or both');
10
+
11
+ /** A filter's value on the wire: exact values, a range, or ranges any of which matches */
12
+ export const filterValueSchema = z.union([z.array(z.string()), rangeSchema, z.array(rangeSchema).min(1)]);
13
+
14
+ const pointSchema = z.object({ lat: z.number().min(-90).max(90), lng: z.number().min(-180).max(180) });
15
+
16
+ export const boundingBoxSchema = z.object({ topRight: pointSchema, bottomLeft: pointSchema });
17
+
18
+ type WireRange = z.infer<typeof rangeSchema>;
19
+ type FilterValue = z.infer<typeof filterValueSchema>;
20
+ type FilterType = BackendSearchCapability['filters'][string]['type'];
21
+ export type Violation = { field: string; message: string; value?: unknown };
22
+ type Normalised = { range: RangeFilter } | { problem: string };
23
+
24
+ const FILTER_DATE = /^\d{4}-\d{2}-\d{2}(?:[Tt ]|$)/;
25
+ const DATE_FORMS = 'a YYYY-MM-DD date or an RFC 3339 date-time';
26
+
27
+ /** Filters take a calendar date, covering its whole UTC day, or a date-time */
28
+ const filterDateSpan = (value: string) => (FILTER_DATE.test(value) ? dateSpan(value) : undefined);
29
+
30
+ const toBound = (type: 'date' | 'number', side: 'gte' | 'lte', value: string | number): RangeBound | undefined => {
31
+ if (type === 'number') {
32
+ return typeof value === 'number' ? value : undefined;
33
+ }
34
+
35
+ const span = typeof value === 'string' ? filterDateSpan(value) : undefined;
36
+
37
+ return span && (side === 'gte' ? span.first : span.last);
38
+ };
39
+
40
+ const normaliseRange = (type: 'date' | 'number', range: WireRange): Normalised => {
41
+ const normalised: RangeFilter = {};
42
+ for (const side of ['gte', 'lte'] as const) {
43
+ const value = range[side];
44
+ if (value === undefined) {
45
+ continue;
46
+ }
47
+
48
+ const bound = toBound(type, side, value);
49
+ if (bound === undefined) {
50
+ return { problem: `${side} must be ${type === 'date' ? DATE_FORMS : 'a number'}` };
51
+ }
52
+ normalised[side] = bound;
53
+ }
54
+
55
+ const { gte, lte } = normalised;
56
+ if (gte !== undefined && lte !== undefined && Number(gte) > Number(lte)) {
57
+ return { problem: 'has gte above lte' };
58
+ }
59
+
60
+ return { range: normalised };
61
+ };
62
+
63
+ const exactRange = (type: 'date' | 'number', value: string): Normalised => {
64
+ if (type === 'number') {
65
+ const number = Number(value);
66
+
67
+ return value.trim() !== '' && Number.isFinite(number)
68
+ ? { range: { gte: number, lte: number } }
69
+ : { problem: `values must be numbers, not ${JSON.stringify(value)}` };
70
+ }
71
+
72
+ const span = filterDateSpan(value);
73
+
74
+ return span
75
+ ? { range: { gte: span.first, lte: span.last } }
76
+ : { problem: `values must be ${DATE_FORMS}, not ${JSON.stringify(value)}` };
77
+ };
78
+
79
+ const isExact = (value: FilterValue): value is string[] =>
80
+ Array.isArray(value) && value.every((item) => typeof item === 'string');
81
+
82
+ const normalise = (type: FilterType, value: FilterValue): { range: RangeFilter[] } | { problem: string } => {
83
+ if (type !== 'date' && type !== 'number') {
84
+ return { problem: `is a ${type} filter, which takes exact values, not ranges` };
85
+ }
86
+
87
+ const normalised = isExact(value)
88
+ ? value.map((item) => exactRange(type, item))
89
+ : (Array.isArray(value) ? value : [value]).map((range) => normaliseRange(type, range));
90
+ const failed = normalised.find((item) => 'problem' in item);
91
+
92
+ return failed ?? { range: normalised.map((item) => (item as { range: RangeFilter }).range) };
93
+ };
94
+
95
+ /**
96
+ * Split declared filters into exact terms and ranges: date and number filters become ranges, with dates as UTC instants
97
+ */
98
+ export const splitFilters = (declared: BackendSearchCapability['filters'], filters: Record<string, FilterValue>) => {
99
+ const split: SearchRequest['filters'] = { terms: {}, ranges: {} };
100
+ const violations: Violation[] = [];
101
+
102
+ for (const [key, value] of Object.entries(filters)) {
103
+ if (!Object.hasOwn(declared, key)) {
104
+ violations.push({ field: `/filters/${key}`, message: `${key} isn't a declared filter`, value });
105
+ continue;
106
+ }
107
+
108
+ const { type } = declared[key];
109
+ if ((type === 'string' || type === 'boolean') && isExact(value)) {
110
+ const notBoolean = type === 'boolean' ? value.find((item) => item !== 'true' && item !== 'false') : undefined;
111
+ if (notBoolean === undefined) {
112
+ split.terms[key] = value;
113
+ } else {
114
+ violations.push({
115
+ field: `/filters/${key}`,
116
+ message: `${key} values must be true or false, not ${JSON.stringify(notBoolean)}`,
117
+ value,
118
+ });
119
+ }
120
+ continue;
121
+ }
122
+
123
+ const result = normalise(type, value);
124
+ if ('problem' in result) {
125
+ violations.push({ field: `/filters/${key}`, message: `${key} ${result.problem}`, value });
126
+ } else {
127
+ split.ranges[key] = result.range;
128
+ }
129
+ }
130
+
131
+ return { filters: split, violations };
132
+ };
133
+
134
+ /**
135
+ * A request's filters as one condition: each filter must match, by any of its values or ranges
136
+ */
137
+ export const filtersCondition = ({ terms, ranges }: SearchRequest['filters']): VisibilityCondition => ({
138
+ type: 'and',
139
+ clauses: [
140
+ ...Object.entries(terms).map(([field, values]): VisibilityCondition => ({ type: 'in', field, values })),
141
+ ...Object.entries(ranges).map(
142
+ ([field, filters]): VisibilityCondition => ({
143
+ type: 'or',
144
+ clauses: filters.map((bounds) => ({ type: 'range', field, bounds })),
145
+ }),
146
+ ),
147
+ ],
148
+ });
149
+
150
+ /**
151
+ * Problems with a request's geo parameters, given whether the backend supports geo
152
+ */
153
+ export const geoViolations = (
154
+ geo: BackendSearchCapability['geo'],
155
+ { boundingBox, geohashPrecision }: { boundingBox?: BoundingBox; geohashPrecision?: number },
156
+ ): Violation[] => {
157
+ const violations: Violation[] = [];
158
+ if (!geo.supported) {
159
+ const message = "Geo search isn't supported";
160
+ if (boundingBox) {
161
+ violations.push({ field: '/boundingBox', message });
162
+ }
163
+ if (geohashPrecision !== undefined) {
164
+ violations.push({ field: '/geohashPrecision', message });
165
+ }
166
+
167
+ return violations;
168
+ }
169
+
170
+ if (geohashPrecision !== undefined && !boundingBox) {
171
+ violations.push({ field: '/geohashPrecision', message: 'geohashPrecision needs a boundingBox' });
172
+ }
173
+ if (boundingBox && boundingBox.bottomLeft.lat > boundingBox.topRight.lat) {
174
+ violations.push({ field: '/boundingBox', message: 'bottomLeft.lat is north of topRight.lat' });
175
+ }
176
+
177
+ return violations;
178
+ };
@@ -0,0 +1,193 @@
1
+ import {
2
+ DATE_FORMS,
3
+ type FieldDeclaration,
4
+ type FieldDeclarations,
5
+ type FieldType,
6
+ type Scalar,
7
+ storeScalar,
8
+ withId,
9
+ } from './fields.js';
10
+
11
+ type ValueOf<Type extends FieldType> = Type extends 'number'
12
+ ? number
13
+ : Type extends 'boolean'
14
+ ? boolean
15
+ : Type extends 'date'
16
+ ? string | Date
17
+ : string;
18
+
19
+ type Equality<Value> = {
20
+ equals?: Value | null;
21
+ not?: Value | null;
22
+ in?: Value[];
23
+ notIn?: Value[];
24
+ };
25
+
26
+ type Ordered<Value> = { lt?: Value; lte?: Value; gt?: Value; gte?: Value };
27
+
28
+ type ConditionOf<Type extends FieldType> = Type extends 'date' | 'number'
29
+ ? ValueOf<Type> | null | (Equality<ValueOf<Type>> & Ordered<ValueOf<Type>>)
30
+ : ValueOf<Type> | null | Equality<ValueOf<Type>>;
31
+
32
+ type AnyCondition = ConditionOf<Exclude<FieldType, 'geo'>>;
33
+
34
+ type TypedWhere<Fields extends FieldDeclarations> = {
35
+ AND?: TypedWhere<Fields>[];
36
+ OR?: TypedWhere<Fields>[];
37
+ NOT?: TypedWhere<Fields>[];
38
+ id?: ConditionOf<'keyword'>;
39
+ } & {
40
+ -readonly [Name in keyof Fields as Fields[Name]['type'] extends 'geo' ? never : Name]?: ConditionOf<
41
+ Fields[Name]['type']
42
+ >;
43
+ };
44
+
45
+ type UntypedWhere = {
46
+ AND?: UntypedWhere[];
47
+ OR?: UntypedWhere[];
48
+ NOT?: UntypedWhere[];
49
+ [name: string]: AnyCondition | UntypedWhere[] | undefined;
50
+ };
51
+
52
+ /**
53
+ * A search visibility filter's condition over `id` and the declared fields, which every search backend applies
54
+ */
55
+ export type SearchVisibilityWhere<Fields extends FieldDeclarations = FieldDeclarations> = string extends keyof Fields
56
+ ? UntypedWhere
57
+ : TypedWhere<Fields>;
58
+
59
+ /**
60
+ * A checked search visibility filter, with dates as UTC instants
61
+ */
62
+ export type VisibilityCondition =
63
+ | { type: 'and' | 'or'; clauses: VisibilityCondition[] }
64
+ | { type: 'not'; clause: VisibilityCondition }
65
+ | { type: 'in'; field: string; values: Scalar[] }
66
+ | { type: 'present'; field: string }
67
+ | { type: 'range'; field: string; bounds: Partial<Record<'lt' | 'lte' | 'gt' | 'gte', number | Date>> };
68
+
69
+ const COMBINATORS = ['AND', 'OR', 'NOT'] as const;
70
+ const EQUALITY = ['equals', 'not', 'in', 'notIn'];
71
+ const RANGE = ['lt', 'lte', 'gt', 'gte'] as const;
72
+
73
+ const isPlainObject = (value: unknown): value is Record<string, unknown> => {
74
+ if (typeof value !== 'object' || value === null) {
75
+ return false;
76
+ }
77
+ const prototype = Object.getPrototypeOf(value);
78
+
79
+ return prototype === Object.prototype || prototype === null;
80
+ };
81
+
82
+ /**
83
+ * Check a search visibility filter against the field declarations, throwing an `Error` naming the field on any fault
84
+ */
85
+ export const checkSearchVisibility = (fields: FieldDeclarations, where: unknown): VisibilityCondition => {
86
+ const declarations = withId(fields);
87
+ const fail = (key: string, problem: string): never => {
88
+ throw new Error(`Search visibility filter: "${key}" ${problem}`);
89
+ };
90
+
91
+ const scalar = (field: string, type: FieldType, value: unknown): Scalar => {
92
+ if (type === 'date' && value instanceof Date && !Number.isNaN(value.getTime())) {
93
+ return value;
94
+ }
95
+
96
+ const stored = storeScalar(type, value);
97
+ if ('problem' in stored) {
98
+ const problem = type === 'date' ? `must be a Date or ${DATE_FORMS}` : stored.problem;
99
+
100
+ return fail(field, `${problem}, not ${JSON.stringify(value)}`);
101
+ }
102
+
103
+ return stored.value as Scalar;
104
+ };
105
+
106
+ const equals = (field: string, type: FieldType, value: unknown): VisibilityCondition =>
107
+ value === null
108
+ ? { type: 'not', clause: { type: 'present', field } }
109
+ : { type: 'in', field, values: [scalar(field, type, value)] };
110
+
111
+ const list = (field: string, type: FieldType, operator: string, value: unknown): VisibilityCondition => {
112
+ if (!Array.isArray(value)) {
113
+ return fail(field, `takes an array for ${operator}`);
114
+ }
115
+
116
+ return { type: 'in', field, values: value.map((item) => scalar(field, type, item)) };
117
+ };
118
+
119
+ const condition = (field: string, declaration: FieldDeclaration, value: unknown): VisibilityCondition => {
120
+ const { type } = declaration;
121
+ if (type === 'geo') {
122
+ return fail(field, "is a geo field, which a search visibility filter can't use");
123
+ }
124
+ if (!isPlainObject(value)) {
125
+ return equals(field, type, value);
126
+ }
127
+
128
+ const ordered = type === 'date' || type === 'number';
129
+ const clauses: VisibilityCondition[] = [];
130
+ const bounds: Extract<VisibilityCondition, { type: 'range' }>['bounds'] = {};
131
+ for (const [operator, operand] of Object.entries(value)) {
132
+ if (operand === undefined) {
133
+ continue;
134
+ }
135
+ if (ordered && (RANGE as readonly string[]).includes(operator)) {
136
+ bounds[operator as (typeof RANGE)[number]] = scalar(field, type, operand) as number | Date;
137
+ continue;
138
+ }
139
+ if (!EQUALITY.includes(operator)) {
140
+ return fail(field, `is a ${type} field, which can't take ${operator}`);
141
+ }
142
+
143
+ const positive =
144
+ operator === 'equals' || operator === 'not'
145
+ ? equals(field, type, operand)
146
+ : list(field, type, operator, operand);
147
+ clauses.push(operator === 'not' || operator === 'notIn' ? { type: 'not', clause: positive } : positive);
148
+ }
149
+ if (Object.keys(bounds).length > 0) {
150
+ clauses.push({ type: 'range', field, bounds });
151
+ }
152
+
153
+ return clauses.length === 1 ? clauses[0] : { type: 'and', clauses };
154
+ };
155
+
156
+ const check = (node: unknown, path: string): VisibilityCondition => {
157
+ if (!isPlainObject(node)) {
158
+ return fail(path, 'must be an object of field conditions');
159
+ }
160
+
161
+ const clauses: VisibilityCondition[] = [];
162
+ for (const [key, value] of Object.entries(node)) {
163
+ if (value === undefined) {
164
+ continue;
165
+ }
166
+
167
+ if ((COMBINATORS as readonly string[]).includes(key)) {
168
+ if (!Array.isArray(value)) {
169
+ return fail(key, 'takes an array of filters');
170
+ }
171
+ const nested = value.map((item, index) => check(item, `${key}[${index}]`));
172
+ clauses.push(
173
+ key === 'AND'
174
+ ? { type: 'and', clauses: nested }
175
+ : key === 'OR'
176
+ ? { type: 'or', clauses: nested }
177
+ : { type: 'not', clause: { type: 'or', clauses: nested } },
178
+ );
179
+ continue;
180
+ }
181
+
182
+ const declaration = Object.hasOwn(declarations, key) ? declarations[key] : undefined;
183
+ if (!declaration) {
184
+ return fail(key, "isn't a declared field");
185
+ }
186
+ clauses.push(condition(key, declaration, value));
187
+ }
188
+
189
+ return clauses.length === 1 ? clauses[0] : { type: 'and', clauses };
190
+ };
191
+
192
+ return check(where, 'filter');
193
+ };