arocapi 4.0.1 → 5.0.0-beta.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (242) hide show
  1. package/LICENSE.md +1 -1
  2. package/README.md +37 -781
  3. package/dist/app.d.ts +38 -21
  4. package/dist/app.js +63 -45
  5. package/dist/app.js.map +1 -1
  6. package/dist/routes/capabilities.d.ts +7 -0
  7. package/dist/routes/capabilities.js +13 -0
  8. package/dist/routes/capabilities.js.map +1 -0
  9. package/dist/routes/entities.d.ts +5 -2
  10. package/dist/routes/entities.js +38 -58
  11. package/dist/routes/entities.js.map +1 -1
  12. package/dist/routes/entity.d.ts +7 -2
  13. package/dist/routes/entity.js +14 -30
  14. package/dist/routes/entity.js.map +1 -1
  15. package/dist/routes/file.d.ts +8 -2
  16. package/dist/routes/file.js +38 -73
  17. package/dist/routes/file.js.map +1 -1
  18. package/dist/routes/files.d.ts +5 -2
  19. package/dist/routes/files.js +25 -40
  20. package/dist/routes/files.js.map +1 -1
  21. package/dist/routes/metadata.d.ts +15 -0
  22. package/dist/routes/metadata.js +38 -0
  23. package/dist/routes/metadata.js.map +1 -0
  24. package/dist/routes/search.d.ts +8 -7
  25. package/dist/routes/search.js +91 -120
  26. package/dist/routes/search.js.map +1 -1
  27. package/dist/search/advanced.d.ts +8 -0
  28. package/dist/search/advanced.js +198 -0
  29. package/dist/search/advanced.js.map +1 -0
  30. package/dist/search/backend.d.ts +95 -0
  31. package/dist/search/backend.js +5 -0
  32. package/dist/search/backend.js.map +1 -0
  33. package/dist/search/fields.d.ts +63 -0
  34. package/dist/search/fields.js +203 -0
  35. package/dist/search/fields.js.map +1 -0
  36. package/dist/search/opensearch/advanced.d.ts +4 -0
  37. package/dist/search/opensearch/advanced.js +28 -0
  38. package/dist/search/opensearch/advanced.js.map +1 -0
  39. package/dist/search/opensearch/opensearch.d.ts +16 -0
  40. package/dist/search/opensearch/opensearch.js +187 -0
  41. package/dist/search/opensearch/opensearch.js.map +1 -0
  42. package/dist/search/opensearch/visibility.d.ts +3 -0
  43. package/dist/search/opensearch/visibility.js +27 -0
  44. package/dist/search/opensearch/visibility.js.map +1 -0
  45. package/dist/search/postgres/advanced.d.ts +7 -0
  46. package/dist/search/postgres/advanced.js +54 -0
  47. package/dist/search/postgres/advanced.js.map +1 -0
  48. package/dist/search/postgres/facets.d.ts +2 -0
  49. package/dist/search/postgres/facets.js +27 -0
  50. package/dist/search/postgres/facets.js.map +1 -0
  51. package/dist/search/postgres/geo.d.ts +6 -0
  52. package/dist/search/postgres/geo.js +42 -0
  53. package/dist/search/postgres/geo.js.map +1 -0
  54. package/dist/search/postgres/postgres.d.ts +19 -0
  55. package/dist/search/postgres/postgres.js +157 -0
  56. package/dist/search/postgres/postgres.js.map +1 -0
  57. package/dist/search/postgres/values.d.ts +8 -0
  58. package/dist/search/postgres/values.js +20 -0
  59. package/dist/search/postgres/values.js.map +1 -0
  60. package/dist/search/postgres/visibility.d.ts +4 -0
  61. package/dist/search/postgres/visibility.js +38 -0
  62. package/dist/search/postgres/visibility.js.map +1 -0
  63. package/dist/search/requestFilters.d.ts +39 -0
  64. package/dist/search/requestFilters.js +125 -0
  65. package/dist/search/requestFilters.js.map +1 -0
  66. package/dist/search/visibility.d.ts +51 -0
  67. package/dist/search/visibility.js +100 -0
  68. package/dist/search/visibility.js.map +1 -0
  69. package/dist/storage/backend.d.ts +58 -0
  70. package/dist/storage/backend.js +2 -0
  71. package/dist/storage/backend.js.map +1 -0
  72. package/dist/storage/deliver.d.ts +12 -0
  73. package/dist/storage/deliver.js +34 -0
  74. package/dist/storage/deliver.js.map +1 -0
  75. package/dist/storage/directory/directory.d.ts +6 -0
  76. package/dist/storage/directory/directory.js +24 -0
  77. package/dist/storage/directory/directory.js.map +1 -0
  78. package/dist/storage/entity-meta/entity-meta.d.ts +5 -0
  79. package/dist/storage/entity-meta/entity-meta.js +14 -0
  80. package/dist/storage/entity-meta/entity-meta.js.map +1 -0
  81. package/dist/storage/location.d.ts +19 -0
  82. package/dist/storage/location.js +36 -0
  83. package/dist/storage/location.js.map +1 -0
  84. package/dist/storage/ocfl/ocfl.d.ts +6 -0
  85. package/dist/storage/ocfl/ocfl.js +45 -0
  86. package/dist/storage/ocfl/ocfl.js.map +1 -0
  87. package/dist/storage/results.d.ts +7 -0
  88. package/dist/storage/results.js +22 -0
  89. package/dist/storage/results.js.map +1 -0
  90. package/dist/storage/s3/s3.d.ts +10 -0
  91. package/dist/storage/s3/s3.js +47 -0
  92. package/dist/storage/s3/s3.js.map +1 -0
  93. package/dist/transformers/default.d.ts +11 -22
  94. package/dist/transformers/default.js +41 -15
  95. package/dist/transformers/default.js.map +1 -1
  96. package/dist/types/access.d.ts +9 -0
  97. package/dist/types/access.js +2 -0
  98. package/dist/types/access.js.map +1 -0
  99. package/dist/types/context.d.ts +6 -0
  100. package/dist/types/context.js +2 -0
  101. package/dist/types/context.js.map +1 -0
  102. package/dist/types/transformers.d.ts +4 -10
  103. package/dist/types/visibility.d.ts +6 -0
  104. package/dist/types/visibility.js +2 -0
  105. package/dist/types/visibility.js.map +1 -0
  106. package/dist/utils/access.d.ts +17 -0
  107. package/dist/utils/access.js +41 -0
  108. package/dist/utils/access.js.map +1 -0
  109. package/dist/utils/errors.d.ts +19 -10
  110. package/dist/utils/errors.js +44 -8
  111. package/dist/utils/errors.js.map +1 -1
  112. package/dist/utils/headers.d.ts +1 -1
  113. package/dist/utils/headers.js +10 -1
  114. package/dist/utils/headers.js.map +1 -1
  115. package/dist/utils/validation.d.ts +2 -0
  116. package/dist/utils/validation.js +8 -0
  117. package/dist/utils/validation.js.map +1 -0
  118. package/dist/utils/visibility.d.ts +18 -0
  119. package/dist/utils/visibility.js +16 -0
  120. package/dist/utils/visibility.js.map +1 -0
  121. package/package.json +84 -45
  122. package/prisma/models/entity.prisma +9 -8
  123. package/prisma/models/entitySearch.prisma +21 -0
  124. package/prisma/models/file.prisma +3 -3
  125. package/src/app.ts +169 -102
  126. package/src/routes/capabilities.ts +22 -0
  127. package/src/routes/entities.ts +49 -68
  128. package/src/routes/entity.ts +22 -36
  129. package/src/routes/file.ts +54 -104
  130. package/src/routes/files.ts +32 -48
  131. package/src/routes/metadata.ts +67 -0
  132. package/src/routes/search.ts +132 -152
  133. package/src/search/advanced.ts +258 -0
  134. package/src/search/backend.ts +91 -0
  135. package/src/search/fields.ts +320 -0
  136. package/src/search/opensearch/advanced.ts +36 -0
  137. package/src/search/opensearch/opensearch.ts +290 -0
  138. package/src/search/opensearch/visibility.ts +38 -0
  139. package/src/search/postgres/advanced.ts +76 -0
  140. package/src/search/postgres/facets.ts +34 -0
  141. package/src/search/postgres/geo.ts +58 -0
  142. package/src/search/postgres/postgres.ts +232 -0
  143. package/src/search/postgres/values.ts +33 -0
  144. package/src/search/postgres/visibility.ts +54 -0
  145. package/src/search/requestFilters.ts +178 -0
  146. package/src/search/visibility.ts +193 -0
  147. package/src/storage/backend.ts +75 -0
  148. package/src/storage/deliver.ts +65 -0
  149. package/src/storage/directory/directory.ts +39 -0
  150. package/src/storage/entity-meta/entity-meta.ts +25 -0
  151. package/src/storage/location.ts +78 -0
  152. package/src/storage/ocfl/ocfl.ts +71 -0
  153. package/src/storage/results.ts +32 -0
  154. package/src/storage/s3/s3.ts +79 -0
  155. package/src/transformers/default.ts +115 -85
  156. package/src/types/access.ts +25 -0
  157. package/src/types/context.ts +15 -0
  158. package/src/types/transformers.ts +8 -34
  159. package/src/types/visibility.ts +21 -0
  160. package/src/utils/access.ts +80 -0
  161. package/src/utils/errors.ts +68 -11
  162. package/src/utils/headers.ts +22 -2
  163. package/src/utils/validation.ts +11 -0
  164. package/src/utils/visibility.ts +56 -0
  165. package/dist/app.test.d.ts +0 -1
  166. package/dist/app.test.js +0 -89
  167. package/dist/app.test.js.map +0 -1
  168. package/dist/express.d.ts +0 -4
  169. package/dist/express.js +0 -14
  170. package/dist/express.js.map +0 -1
  171. package/dist/index.dev.d.ts +0 -1
  172. package/dist/index.dev.js +0 -156
  173. package/dist/index.dev.js.map +0 -1
  174. package/dist/routes/crate.d.ts +0 -9
  175. package/dist/routes/crate.js +0 -82
  176. package/dist/routes/crate.js.map +0 -1
  177. package/dist/routes/crate.test.d.ts +0 -1
  178. package/dist/routes/crate.test.js +0 -342
  179. package/dist/routes/crate.test.js.map +0 -1
  180. package/dist/routes/entities.test.d.ts +0 -1
  181. package/dist/routes/entities.test.js +0 -273
  182. package/dist/routes/entities.test.js.map +0 -1
  183. package/dist/routes/entity.test.d.ts +0 -1
  184. package/dist/routes/entity.test.js +0 -130
  185. package/dist/routes/entity.test.js.map +0 -1
  186. package/dist/routes/file.test.d.ts +0 -1
  187. package/dist/routes/file.test.js +0 -315
  188. package/dist/routes/file.test.js.map +0 -1
  189. package/dist/routes/files.test.d.ts +0 -1
  190. package/dist/routes/files.test.js +0 -222
  191. package/dist/routes/files.test.js.map +0 -1
  192. package/dist/routes/search.test.d.ts +0 -1
  193. package/dist/routes/search.test.js +0 -780
  194. package/dist/routes/search.test.js.map +0 -1
  195. package/dist/test/helpers/fastify.d.ts +0 -9
  196. package/dist/test/helpers/fastify.js +0 -18
  197. package/dist/test/helpers/fastify.js.map +0 -1
  198. package/dist/test/integration.setup.d.ts +0 -29
  199. package/dist/test/integration.setup.js +0 -188
  200. package/dist/test/integration.setup.js.map +0 -1
  201. package/dist/test/integration.test.d.ts +0 -1
  202. package/dist/test/integration.test.js +0 -396
  203. package/dist/test/integration.test.js.map +0 -1
  204. package/dist/transformers/default.test.d.ts +0 -1
  205. package/dist/transformers/default.test.js +0 -163
  206. package/dist/transformers/default.test.js.map +0 -1
  207. package/dist/transformers/transformer.test.d.ts +0 -1
  208. package/dist/transformers/transformer.test.js +0 -187
  209. package/dist/transformers/transformer.test.js.map +0 -1
  210. package/dist/types/fileHandlers.d.ts +0 -41
  211. package/dist/types/fileHandlers.js +0 -2
  212. package/dist/types/fileHandlers.js.map +0 -1
  213. package/dist/utils/errors.test.d.ts +0 -1
  214. package/dist/utils/errors.test.js +0 -66
  215. package/dist/utils/errors.test.js.map +0 -1
  216. package/dist/utils/queryBuilder.d.ts +0 -35
  217. package/dist/utils/queryBuilder.js +0 -98
  218. package/dist/utils/queryBuilder.js.map +0 -1
  219. package/prisma.config.ts +0 -9
  220. package/src/app.test.ts +0 -119
  221. package/src/express.ts +0 -21
  222. package/src/index.dev.ts +0 -200
  223. package/src/reset.d.ts +0 -1
  224. package/src/routes/__snapshots__/entities.test.ts.snap +0 -96
  225. package/src/routes/__snapshots__/entity.test.ts.snap +0 -74
  226. package/src/routes/__snapshots__/search.test.ts.snap +0 -165
  227. package/src/routes/crate.test.ts +0 -423
  228. package/src/routes/crate.ts +0 -123
  229. package/src/routes/entities.test.ts +0 -320
  230. package/src/routes/entity.test.ts +0 -160
  231. package/src/routes/file.test.ts +0 -395
  232. package/src/routes/files.test.ts +0 -278
  233. package/src/routes/search.test.ts +0 -903
  234. package/src/test/__snapshots__/integration.test.ts.snap +0 -18
  235. package/src/test/helpers/fastify.ts +0 -27
  236. package/src/test/integration.setup.ts +0 -210
  237. package/src/test/integration.test.ts +0 -482
  238. package/src/transformers/default.test.ts +0 -184
  239. package/src/transformers/transformer.test.ts +0 -252
  240. package/src/types/fileHandlers.ts +0 -65
  241. package/src/utils/errors.test.ts +0 -75
  242. package/src/utils/queryBuilder.ts +0 -132
@@ -0,0 +1,76 @@
1
+ import { join, type Sql, sqltag as sql } from '@prisma/client/runtime/client';
2
+ import { matchedDateSpan } from '../advanced.js';
3
+ import { type AdvancedQuery, SearchQueryError } from '../backend.js';
4
+ import type { FieldDeclarations, FieldType } from '../fields.js';
5
+ import { fieldValue, parameter, TEXT_SEARCH_CONFIG, valuesOf } from './values.js';
6
+
7
+ // A tsquery's lexemes as text: single-quoted, with quotes doubled
8
+ const LEXEME = "'((?:[^']|'')*)'";
9
+
10
+ const exactValue = (field: string, type: string, value: string) => {
11
+ if (type === 'boolean' && value !== 'true' && value !== 'false') {
12
+ throw new SearchQueryError(`"${field}" is a boolean field: "${value}" isn't true or false`);
13
+ }
14
+
15
+ return type === 'boolean' ? value === 'true' : value;
16
+ };
17
+
18
+ const exactCondition = (field: string, type: FieldType, value: string) => {
19
+ if (type === 'date') {
20
+ const { first, last } = matchedDateSpan(field, value);
21
+
22
+ return sql`${fieldValue(type)} BETWEEN ${parameter(type, first)} AND ${parameter(type, last)}`;
23
+ }
24
+
25
+ return sql`${fieldValue(type)} = ${parameter(type, exactValue(field, type, value))}`;
26
+ };
27
+
28
+ /**
29
+ * Translate an advanced query tree to a SQL condition on the `search` row, and a rank over its positive text matches
30
+ * Text values match on the `text` tsvector, whose lexemes are prefixed `<field>.`
31
+ */
32
+ export const advancedQueryFor = (query: AdvancedQuery, fields: FieldDeclarations) => {
33
+ const ranked: Sql[] = [];
34
+
35
+ const tsquery = (field: string, value: string, phrase: boolean) => {
36
+ const parsed = phrase
37
+ ? sql`phraseto_tsquery(${TEXT_SEARCH_CONFIG}, ${value}::text)`
38
+ : sql`plainto_tsquery(${TEXT_SEARCH_CONFIG}, ${value}::text)`;
39
+
40
+ return sql`regexp_replace(${parsed}::text, ${LEXEME}::text, ${`'${field}.\\1'`}::text, 'g')::tsquery`;
41
+ };
42
+
43
+ const translate = (node: AdvancedQuery, negated: boolean): Sql => {
44
+ switch (node.type) {
45
+ case 'and':
46
+ case 'or':
47
+ return sql`(${join(
48
+ node.clauses.map((clause) => translate(clause, negated)),
49
+ node.type === 'and' ? ' AND ' : ' OR ',
50
+ )})`;
51
+ case 'not':
52
+ return sql`NOT ${translate(node.clause, !negated)}`;
53
+ case 'match': {
54
+ const { field, value, phrase } = node;
55
+ const { type } = fields[field];
56
+ if (type === 'text') {
57
+ const matching = tsquery(field, value, phrase);
58
+ if (!negated) {
59
+ ranked.push(matching);
60
+ }
61
+
62
+ return sql`search.text @@ ${matching}`;
63
+ }
64
+
65
+ return sql`EXISTS (SELECT FROM ${valuesOf(field)} WHERE ${exactCondition(field, type, value)})`;
66
+ }
67
+ default:
68
+ return sql`TRUE`;
69
+ }
70
+ };
71
+
72
+ const condition = translate(query, false);
73
+ const rank = ranked.length > 0 ? sql`ts_rank(search.text, ${join(ranked, ' || ')})` : sql`0`;
74
+
75
+ return { condition, rank };
76
+ };
@@ -0,0 +1,34 @@
1
+ import { sqltag as sql } from '@prisma/client/runtime/client';
2
+ import { fieldValue, valuesOf } from './values.js';
3
+
4
+ /**
5
+ * A facet's buckets over the `matched` rows: the `size` commonest values, most matches first
6
+ */
7
+ export const valueBuckets = (name: string, size: number) => sql`(
8
+ SELECT coalesce(jsonb_agg(
9
+ jsonb_build_object('name', bucket.name, 'count', bucket.count) ORDER BY bucket.count DESC, bucket.name COLLATE "C"
10
+ ), '[]')
11
+ FROM (
12
+ SELECT value.name, count(*)::int AS count
13
+ FROM matched AS search, LATERAL (SELECT DISTINCT field.value AS name FROM ${valuesOf(name)}) AS value
14
+ GROUP BY value.name
15
+ ORDER BY count DESC, value.name COLLATE "C"
16
+ LIMIT ${size}::int
17
+ ) AS bucket
18
+ )`;
19
+
20
+ /**
21
+ * A date facet's buckets over the `matched` rows: one per UTC year with matches, named `YYYY`, in year order
22
+ */
23
+ export const yearBuckets = (name: string) => sql`(
24
+ SELECT coalesce(jsonb_agg(
25
+ jsonb_build_object('name', to_char(bucket.year, 'YYYY'), 'count', bucket.count) ORDER BY bucket.year
26
+ ), '[]')
27
+ FROM (
28
+ SELECT value.year, count(*)::int AS count
29
+ FROM matched AS search, LATERAL (
30
+ SELECT DISTINCT date_trunc('year', ${fieldValue('date')} AT TIME ZONE 'UTC') AS year FROM ${valuesOf(name)}
31
+ ) AS value
32
+ GROUP BY value.year
33
+ ) AS bucket
34
+ )`;
@@ -0,0 +1,58 @@
1
+ import { type Sql, sqltag as sql } from '@prisma/client/runtime/client';
2
+ import { type BoundingBox, GEOHASH_GRID_CELLS } from '../backend.js';
3
+ import type { GeoPoint } from '../fields.js';
4
+
5
+ const GEOHASH_ALPHABET = '0123456789bcdefghjkmnpqrstuvwxyz';
6
+
7
+ /**
8
+ * A point's 12-character geohash
9
+ */
10
+ export const geohash = ({ lat, lng }: GeoPoint) => {
11
+ const ranges = [
12
+ [-180, 180],
13
+ [-90, 90],
14
+ ];
15
+ let hash = '';
16
+ let bits = 0;
17
+ for (let bit = 0; bit < 60; bit++) {
18
+ const axis = bit % 2;
19
+ const range = ranges[axis];
20
+ const middle = (range[0] + range[1]) / 2;
21
+ const upper = (axis === 0 ? lng : lat) >= middle;
22
+ range[upper ? 0 : 1] = middle;
23
+ bits = bits * 2 + Number(upper);
24
+ if (bit % 5 === 4) {
25
+ hash += GEOHASH_ALPHABET[bits];
26
+ bits = 0;
27
+ }
28
+ }
29
+
30
+ return hash;
31
+ };
32
+
33
+ /**
34
+ * A condition that the `search` row's point lies in the box, which may cross the antimeridian
35
+ */
36
+ export const boundingBoxCondition = ({ topRight, bottomLeft }: BoundingBox): Sql => {
37
+ const box = (west: number, east: number) =>
38
+ sql`search.point <@ box(point(${west}::float8, ${bottomLeft.lat}::float8), point(${east}::float8, ${topRight.lat}::float8))`;
39
+
40
+ return bottomLeft.lng <= topRight.lng
41
+ ? box(bottomLeft.lng, topRight.lng)
42
+ : sql`(${box(bottomLeft.lng, 180)} OR ${box(-180, topRight.lng)})`;
43
+ };
44
+
45
+ /**
46
+ * Counts of the `matched` rows with a point, per geohash cell of `precision` characters (at most `GEOHASH_GRID_CELLS`, those with the most)
47
+ */
48
+ export const geohashGrid = (precision: number) => sql`(
49
+ SELECT coalesce(jsonb_object_agg(cell.geohash, cell.count), '{}')
50
+ FROM (
51
+ SELECT left(search.geohash, ${precision}::int) COLLATE "C" AS geohash, count(*)::int AS count
52
+ FROM matched AS search
53
+ WHERE search.geohash IS NOT NULL
54
+ GROUP BY 1
55
+ ORDER BY count DESC, geohash
56
+ LIMIT ${GEOHASH_GRID_CELLS}::int
57
+ ) AS cell
58
+ )`;
@@ -0,0 +1,232 @@
1
+ import { join, type Sql, sqltag as sql } from '@prisma/client/runtime/client';
2
+ import { SearchQueryError, type SearchRequest, type SearchResult } from '../backend.js';
3
+ import {
4
+ capabilitiesFromFields,
5
+ type FieldDeclarations,
6
+ resolveFields,
7
+ type SearchDocument,
8
+ storeDocuments,
9
+ toWire,
10
+ } from '../fields.js';
11
+ import { filtersCondition } from '../requestFilters.js';
12
+ import { checkSearchVisibility } from '../visibility.js';
13
+ import { advancedQueryFor } from './advanced.js';
14
+ import { valueBuckets, yearBuckets } from './facets.js';
15
+ import { boundingBoxCondition, geohash, geohashGrid } from './geo.js';
16
+ import { TEXT_SEARCH_CONFIG, typed } from './values.js';
17
+ import { visibilityCondition } from './visibility.js';
18
+
19
+ /**
20
+ * The Prisma client PostgresBackend needs: any Prisma 7 client on a schema that includes the arocapi models,
21
+ * including the client inside `$transaction`
22
+ */
23
+ export type PostgresClient = {
24
+ $queryRaw<T = unknown>(query: Sql): PromiseLike<T>;
25
+ $executeRaw(query: Sql): PromiseLike<number>;
26
+ };
27
+
28
+ export type PostgresBackendOptions<Fields extends FieldDeclarations> = {
29
+ prisma: PostgresClient;
30
+ fields: Fields;
31
+ };
32
+
33
+ const WEIGHT_LABELS = ['D', 'C', 'B', 'A'];
34
+
35
+ /**
36
+ * Postgres search backend: holds one search document per entity in the `EntitySearch` table
37
+ */
38
+ export const PostgresBackend = <const Fields extends FieldDeclarations>({
39
+ prisma,
40
+ fields,
41
+ }: PostgresBackendOptions<Fields>) => {
42
+ const capabilities = capabilitiesFromFields(fields);
43
+ const { basic, facets } = resolveFields(fields);
44
+ const textFields = Object.keys(fields).filter((name) => fields[name].type === 'text');
45
+
46
+ const basicVectors = basic.map(
47
+ ({ name, weight }) =>
48
+ sql`coalesce(setweight(to_tsvector(${TEXT_SEARCH_CONFIG}, docs."values" -> ${name}), ${WEIGHT_LABELS[weight - 1]}::"char"), '')`,
49
+ );
50
+ const basicVector = basicVectors.length > 0 ? join(basicVectors, ' || ') : sql`''::tsvector`;
51
+
52
+ // Built as text so each field keeps its own positions; tsvector `||` would shift them towards the 16383 limit
53
+ const textVector = sql`(
54
+ SELECT coalesce(string_agg(
55
+ '''' || replace(replace(field || '.' || lexeme, '\\', '\\\\'), '''', '''''') || ''':' || array_to_string(positions, ','),
56
+ ' '
57
+ ), '')::tsvector
58
+ FROM unnest(${textFields}::text[]) AS field, unnest(to_tsvector(${TEXT_SEARCH_CONFIG}, docs."values" -> field))
59
+ )`;
60
+
61
+ const facetBuckets = facets.map(
62
+ ({ name, type, size }) =>
63
+ sql`jsonb_build_object(${name}::text, ${type === 'date' ? yearBuckets(name) : valueBuckets(name, size)})`,
64
+ );
65
+
66
+ const queryFor = (request: SearchRequest) => {
67
+ if (request.searchType === 'advanced') {
68
+ return advancedQueryFor(request.query, fields);
69
+ }
70
+ if (request.query.trim() === '') {
71
+ return { condition: sql`TRUE`, rank: sql`0` };
72
+ }
73
+ const tsquery = sql`websearch_to_tsquery(${TEXT_SEARCH_CONFIG}, ${request.query}::text)`;
74
+
75
+ // A query with no words left after parsing, such as `*`, matches everything
76
+ return {
77
+ condition: sql`(numnode(${tsquery}) = 0 OR search.basic @@ ${tsquery})`,
78
+ rank: sql`ts_rank(search.basic, ${tsquery})`,
79
+ };
80
+ };
81
+
82
+ return {
83
+ capabilities,
84
+
85
+ search: async (request: SearchRequest): Promise<SearchResult> => {
86
+ const { condition: matches, rank } = queryFor(request);
87
+ const conditions = [
88
+ matches,
89
+ visibilityCondition(filtersCondition(request.filters), fields),
90
+ ...(request.boundingBox ? [boundingBoxCondition(request.boundingBox)] : []),
91
+ visibilityCondition(checkSearchVisibility(fields, request.visibility), fields),
92
+ ];
93
+
94
+ const { sort, order, geohashPrecision } = request;
95
+ const [key, direction] =
96
+ sort === 'relevance'
97
+ ? [rank, sql`DESC`]
98
+ : [
99
+ typed(fields[sort].type, sql`search."values" ->> ${sort}::text`),
100
+ order === 'asc' ? sql`ASC NULLS LAST` : sql`DESC NULLS LAST`,
101
+ ];
102
+ const [result] = await asQueryErrors(
103
+ prisma.$queryRaw<(Pick<SearchResult, 'total' | 'facets' | 'geohashGrid'> & { hits: string[] })[]>(sql`
104
+ WITH matched AS (
105
+ SELECT search.id, search."values", search.geohash, ${key} AS key
106
+ FROM entity_search AS search
107
+ WHERE ${join(conditions, ' AND ')}
108
+ )
109
+ SELECT
110
+ (SELECT count(*)::int FROM matched) AS total,
111
+ (
112
+ SELECT coalesce(jsonb_agg(page.id ORDER BY page.key ${direction}, page.id COLLATE "C"), '[]')
113
+ FROM (
114
+ SELECT search.id, search.key
115
+ FROM matched AS search
116
+ ORDER BY search.key ${direction}, search.id COLLATE "C"
117
+ LIMIT ${request.limit}::int OFFSET ${request.offset}::int
118
+ ) AS page
119
+ ) AS hits,
120
+ ${join([sql`'{}'::jsonb`, ...facetBuckets], ' || ')} AS facets,
121
+ ${geohashPrecision === undefined ? sql`NULL` : geohashGrid(geohashPrecision)} AS "geohashGrid"
122
+ `),
123
+ );
124
+
125
+ return {
126
+ total: result.total,
127
+ hits: result.hits.map((id) => ({ id })),
128
+ facets: result.facets,
129
+ ...(result.geohashGrid && { geohashGrid: result.geohashGrid }),
130
+ };
131
+ },
132
+
133
+ /**
134
+ * Upsert search documents, replacing any already indexed with the same id
135
+ * Throws, naming them, if any entities don't exist, and then stores nothing from the batch
136
+ */
137
+ index: async (docs: SearchDocument<Fields>[], { tx }: { tx?: PostgresClient } = {}) => {
138
+ const stored = storeDocuments(fields, docs);
139
+
140
+ const rows = [...new Map(stored.map((doc) => [doc.id, doc])).values()].map(({ id, values, point }) => ({
141
+ id,
142
+ values: Object.fromEntries(
143
+ Object.entries(values).map(([name, value]) => [
144
+ name,
145
+ Array.isArray(value) ? value.map(toWire) : toWire(value),
146
+ ]),
147
+ ),
148
+ ...(point && { ...point, geohash: geohash(point) }),
149
+ }));
150
+
151
+ const ids = rows.map(({ id }) => id);
152
+ const upsert = (tx ?? prisma).$queryRaw<{ id: string }[]>(sql`
153
+ WITH docs AS (
154
+ SELECT * FROM jsonb_to_recordset(${JSON.stringify(rows)}::jsonb)
155
+ AS docs(id text, "values" jsonb, lat float8, lng float8, geohash text)
156
+ ),
157
+ missing AS (
158
+ SELECT docs.id FROM docs WHERE NOT EXISTS (SELECT FROM entity WHERE entity.id = docs.id)
159
+ ),
160
+ upserted AS (
161
+ INSERT INTO entity_search (id, "values", basic, text, point, geohash)
162
+ SELECT docs.id, docs."values", ${basicVector}, ${textVector}, point(docs.lng, docs.lat), docs.geohash
163
+ FROM docs
164
+ WHERE NOT EXISTS (SELECT FROM missing)
165
+ ON CONFLICT (id) DO UPDATE SET
166
+ "values" = excluded."values",
167
+ basic = excluded.basic,
168
+ text = excluded.text,
169
+ point = excluded.point,
170
+ geohash = excluded.geohash
171
+ )
172
+ SELECT id FROM missing ORDER BY id
173
+ `);
174
+
175
+ let missing: { id: string }[];
176
+ try {
177
+ missing = await upsert;
178
+ } catch (error) {
179
+ if (driverErrorOf(error)?.originalCode !== FOREIGN_KEY_VIOLATION) {
180
+ throw error;
181
+ }
182
+
183
+ // An entity was deleted after the check; `tx` is aborted, so look outside it
184
+ missing = await prisma.$queryRaw<{ id: string }[]>(sql`
185
+ SELECT doc.id FROM unnest(${ids}::text[]) AS doc(id)
186
+ WHERE NOT EXISTS (SELECT FROM entity WHERE entity.id = doc.id)
187
+ ORDER BY doc.id
188
+ `);
189
+ if (missing.length === 0) {
190
+ throw error;
191
+ }
192
+ }
193
+
194
+ if (missing.length > 0) {
195
+ throw new Error(
196
+ `Search documents reference entities that don't exist: ${missing.map(({ id }) => JSON.stringify(id)).join(', ')}`,
197
+ );
198
+ }
199
+ },
200
+
201
+ /**
202
+ * Remove search documents, ignoring ids that aren't indexed
203
+ */
204
+ remove: async (ids: string[]) => {
205
+ await prisma.$executeRaw(sql`DELETE FROM entity_search WHERE id = ANY(${ids}::text[])`);
206
+ },
207
+ };
208
+ };
209
+
210
+ type DriverAdapterError = {
211
+ meta?: { driverAdapterError?: { cause?: { originalCode?: string; originalMessage?: string } } };
212
+ };
213
+
214
+ const driverErrorOf = (error: unknown) => (error as DriverAdapterError).meta?.driverAdapterError?.cause;
215
+
216
+ const FOREIGN_KEY_VIOLATION = '23503';
217
+
218
+ // Data exceptions (22) come from query values a field can't hold; program limits (54) from queries too big to run
219
+ const QUERY_ERROR_CLASSES = ['22', '54'];
220
+
221
+ const asQueryErrors = async <T>(query: PromiseLike<T>) => {
222
+ try {
223
+ return await query;
224
+ } catch (error) {
225
+ const cause = driverErrorOf(error);
226
+ if (cause && QUERY_ERROR_CLASSES.some((errorClass) => cause.originalCode?.startsWith(errorClass))) {
227
+ throw new SearchQueryError(cause.originalMessage);
228
+ }
229
+
230
+ throw error;
231
+ }
232
+ };
@@ -0,0 +1,33 @@
1
+ import { type Sql, sqltag as sql } from '@prisma/client/runtime/client';
2
+ import { type FieldType, type Scalar, toWire } from '../fields.js';
3
+
4
+ export const TEXT_SEARCH_CONFIG = sql`'simple'::regconfig`;
5
+
6
+ /** A JSON path to a field's values in a search row: none when missing, one per array item */
7
+ export const valuesPath = (field: string) => `$."${field}"[*]`;
8
+
9
+ /**
10
+ * A `FROM` item of a search row's values for a field, as `field.value` text
11
+ */
12
+ export const valuesOf = (field: string) =>
13
+ sql`LATERAL (SELECT jsonb_path_query(search."values", ${valuesPath(field)}::jsonpath) #>> '{}' AS value) AS field`;
14
+
15
+ /**
16
+ * Text as the field's type, comparing as OpenSearch does: strings in code point order
17
+ */
18
+ export const typed = (type: FieldType, text: Sql) => {
19
+ switch (type) {
20
+ case 'number':
21
+ return sql`(${text})::float8`;
22
+ case 'date':
23
+ return sql`(${text})::timestamptz`;
24
+ case 'boolean':
25
+ return sql`(${text})::boolean`;
26
+ default:
27
+ return sql`(${text}) COLLATE "C"`;
28
+ }
29
+ };
30
+
31
+ export const fieldValue = (type: FieldType) => typed(type, sql`field.value`);
32
+
33
+ export const parameter = (type: FieldType, value: Scalar) => typed(type, sql`${String(toWire(value))}::text`);
@@ -0,0 +1,54 @@
1
+ import { join, type Sql, sqltag as sql } from '@prisma/client/runtime/client';
2
+ import { type FieldDeclarations, type FieldType, type Scalar, toWire } from '../fields.js';
3
+ import type { VisibilityCondition } from '../visibility.js';
4
+ import { fieldValue, parameter, valuesOf, valuesPath } from './values.js';
5
+
6
+ const OPERATORS = { lt: sql`<`, lte: sql`<=`, gt: sql`>`, gte: sql`>=` };
7
+
8
+ // Boolean filter values arrive as text
9
+ const literal = (type: FieldType, value: Scalar) =>
10
+ JSON.stringify(type === 'boolean' && (value === 'true' || value === 'false') ? value === 'true' : toWire(value));
11
+
12
+ /**
13
+ * Translate a checked search visibility filter to a parameterised SQL condition on the `search` row
14
+ * Equality is JSON path containment, so the `values` index can serve it
15
+ */
16
+ export const visibilityCondition = (condition: VisibilityCondition, fields: FieldDeclarations): Sql => {
17
+ const translate = (node: VisibilityCondition): Sql => {
18
+ switch (node.type) {
19
+ case 'and':
20
+ return sql`(${join([sql`TRUE`, ...node.clauses.map(translate)], ' AND ')})`;
21
+ case 'or':
22
+ return sql`(${join([sql`FALSE`, ...node.clauses.map(translate)], ' OR ')})`;
23
+ case 'not':
24
+ return sql`NOT ${translate(node.clause)}`;
25
+ case 'present':
26
+ return node.field === 'id' ? sql`TRUE` : sql`search."values" @? ${valuesPath(node.field)}::jsonpath`;
27
+ case 'in': {
28
+ const { field, values } = node;
29
+ if (field === 'id') {
30
+ return sql`search.id = ANY(${values.map(String)}::text[])`;
31
+ }
32
+ if (values.length === 0) {
33
+ return sql`FALSE`;
34
+ }
35
+
36
+ const { type } = fields[field];
37
+ const equals = values.map((value) => `@ == ${literal(type, value)}`).join(' || ');
38
+
39
+ return sql`search."values" @? ${`${valuesPath(field)} ? (${equals})`}::jsonpath`;
40
+ }
41
+ case 'range': {
42
+ const { type } = fields[node.field];
43
+ const bounds = Object.entries(node.bounds).map(
44
+ ([bound, value]) =>
45
+ sql`${fieldValue(type)} ${OPERATORS[bound as keyof typeof OPERATORS]} ${parameter(type, value)}`,
46
+ );
47
+
48
+ return sql`EXISTS (SELECT FROM ${valuesOf(node.field)} WHERE ${join(bounds, ' AND ')})`;
49
+ }
50
+ }
51
+ };
52
+
53
+ return translate(condition);
54
+ };
@@ -0,0 +1,178 @@
1
+ import { z } from 'zod';
2
+ import type { BackendSearchCapability, BoundingBox, RangeBound, RangeFilter, SearchRequest } from './backend.js';
3
+ import { dateSpan } from './fields.js';
4
+ import type { VisibilityCondition } from './visibility.js';
5
+
6
+ const boundSchema = z.union([z.string(), z.number()]);
7
+ const rangeSchema = z
8
+ .strictObject({ gte: boundSchema.optional(), lte: boundSchema.optional() })
9
+ .refine(({ gte, lte }) => gte !== undefined || lte !== undefined, 'A range needs gte, lte or both');
10
+
11
+ /** A filter's value on the wire: exact values, a range, or ranges any of which matches */
12
+ export const filterValueSchema = z.union([z.array(z.string()), rangeSchema, z.array(rangeSchema).min(1)]);
13
+
14
+ const pointSchema = z.object({ lat: z.number().min(-90).max(90), lng: z.number().min(-180).max(180) });
15
+
16
+ export const boundingBoxSchema = z.object({ topRight: pointSchema, bottomLeft: pointSchema });
17
+
18
+ type WireRange = z.infer<typeof rangeSchema>;
19
+ type FilterValue = z.infer<typeof filterValueSchema>;
20
+ type FilterType = BackendSearchCapability['filters'][string]['type'];
21
+ export type Violation = { field: string; message: string; value?: unknown };
22
+ type Normalised = { range: RangeFilter } | { problem: string };
23
+
24
+ const FILTER_DATE = /^\d{4}-\d{2}-\d{2}(?:[Tt ]|$)/;
25
+ const DATE_FORMS = 'a YYYY-MM-DD date or an RFC 3339 date-time';
26
+
27
+ /** Filters take a calendar date, covering its whole UTC day, or a date-time */
28
+ const filterDateSpan = (value: string) => (FILTER_DATE.test(value) ? dateSpan(value) : undefined);
29
+
30
+ const toBound = (type: 'date' | 'number', side: 'gte' | 'lte', value: string | number): RangeBound | undefined => {
31
+ if (type === 'number') {
32
+ return typeof value === 'number' ? value : undefined;
33
+ }
34
+
35
+ const span = typeof value === 'string' ? filterDateSpan(value) : undefined;
36
+
37
+ return span && (side === 'gte' ? span.first : span.last);
38
+ };
39
+
40
+ const normaliseRange = (type: 'date' | 'number', range: WireRange): Normalised => {
41
+ const normalised: RangeFilter = {};
42
+ for (const side of ['gte', 'lte'] as const) {
43
+ const value = range[side];
44
+ if (value === undefined) {
45
+ continue;
46
+ }
47
+
48
+ const bound = toBound(type, side, value);
49
+ if (bound === undefined) {
50
+ return { problem: `${side} must be ${type === 'date' ? DATE_FORMS : 'a number'}` };
51
+ }
52
+ normalised[side] = bound;
53
+ }
54
+
55
+ const { gte, lte } = normalised;
56
+ if (gte !== undefined && lte !== undefined && Number(gte) > Number(lte)) {
57
+ return { problem: 'has gte above lte' };
58
+ }
59
+
60
+ return { range: normalised };
61
+ };
62
+
63
+ const exactRange = (type: 'date' | 'number', value: string): Normalised => {
64
+ if (type === 'number') {
65
+ const number = Number(value);
66
+
67
+ return value.trim() !== '' && Number.isFinite(number)
68
+ ? { range: { gte: number, lte: number } }
69
+ : { problem: `values must be numbers, not ${JSON.stringify(value)}` };
70
+ }
71
+
72
+ const span = filterDateSpan(value);
73
+
74
+ return span
75
+ ? { range: { gte: span.first, lte: span.last } }
76
+ : { problem: `values must be ${DATE_FORMS}, not ${JSON.stringify(value)}` };
77
+ };
78
+
79
+ const isExact = (value: FilterValue): value is string[] =>
80
+ Array.isArray(value) && value.every((item) => typeof item === 'string');
81
+
82
+ const normalise = (type: FilterType, value: FilterValue): { range: RangeFilter[] } | { problem: string } => {
83
+ if (type !== 'date' && type !== 'number') {
84
+ return { problem: `is a ${type} filter, which takes exact values, not ranges` };
85
+ }
86
+
87
+ const normalised = isExact(value)
88
+ ? value.map((item) => exactRange(type, item))
89
+ : (Array.isArray(value) ? value : [value]).map((range) => normaliseRange(type, range));
90
+ const failed = normalised.find((item) => 'problem' in item);
91
+
92
+ return failed ?? { range: normalised.map((item) => (item as { range: RangeFilter }).range) };
93
+ };
94
+
95
+ /**
96
+ * Split declared filters into exact terms and ranges: date and number filters become ranges, with dates as UTC instants
97
+ */
98
+ export const splitFilters = (declared: BackendSearchCapability['filters'], filters: Record<string, FilterValue>) => {
99
+ const split: SearchRequest['filters'] = { terms: {}, ranges: {} };
100
+ const violations: Violation[] = [];
101
+
102
+ for (const [key, value] of Object.entries(filters)) {
103
+ if (!Object.hasOwn(declared, key)) {
104
+ violations.push({ field: `/filters/${key}`, message: `${key} isn't a declared filter`, value });
105
+ continue;
106
+ }
107
+
108
+ const { type } = declared[key];
109
+ if ((type === 'string' || type === 'boolean') && isExact(value)) {
110
+ const notBoolean = type === 'boolean' ? value.find((item) => item !== 'true' && item !== 'false') : undefined;
111
+ if (notBoolean === undefined) {
112
+ split.terms[key] = value;
113
+ } else {
114
+ violations.push({
115
+ field: `/filters/${key}`,
116
+ message: `${key} values must be true or false, not ${JSON.stringify(notBoolean)}`,
117
+ value,
118
+ });
119
+ }
120
+ continue;
121
+ }
122
+
123
+ const result = normalise(type, value);
124
+ if ('problem' in result) {
125
+ violations.push({ field: `/filters/${key}`, message: `${key} ${result.problem}`, value });
126
+ } else {
127
+ split.ranges[key] = result.range;
128
+ }
129
+ }
130
+
131
+ return { filters: split, violations };
132
+ };
133
+
134
+ /**
135
+ * A request's filters as one condition: each filter must match, by any of its values or ranges
136
+ */
137
+ export const filtersCondition = ({ terms, ranges }: SearchRequest['filters']): VisibilityCondition => ({
138
+ type: 'and',
139
+ clauses: [
140
+ ...Object.entries(terms).map(([field, values]): VisibilityCondition => ({ type: 'in', field, values })),
141
+ ...Object.entries(ranges).map(
142
+ ([field, filters]): VisibilityCondition => ({
143
+ type: 'or',
144
+ clauses: filters.map((bounds) => ({ type: 'range', field, bounds })),
145
+ }),
146
+ ),
147
+ ],
148
+ });
149
+
150
+ /**
151
+ * Problems with a request's geo parameters, given whether the backend supports geo
152
+ */
153
+ export const geoViolations = (
154
+ geo: BackendSearchCapability['geo'],
155
+ { boundingBox, geohashPrecision }: { boundingBox?: BoundingBox; geohashPrecision?: number },
156
+ ): Violation[] => {
157
+ const violations: Violation[] = [];
158
+ if (!geo.supported) {
159
+ const message = "Geo search isn't supported";
160
+ if (boundingBox) {
161
+ violations.push({ field: '/boundingBox', message });
162
+ }
163
+ if (geohashPrecision !== undefined) {
164
+ violations.push({ field: '/geohashPrecision', message });
165
+ }
166
+
167
+ return violations;
168
+ }
169
+
170
+ if (geohashPrecision !== undefined && !boundingBox) {
171
+ violations.push({ field: '/geohashPrecision', message: 'geohashPrecision needs a boundingBox' });
172
+ }
173
+ if (boundingBox && boundingBox.bottomLeft.lat > boundingBox.topRight.lat) {
174
+ violations.push({ field: '/boundingBox', message: 'bottomLeft.lat is north of topRight.lat' });
175
+ }
176
+
177
+ return violations;
178
+ };