@crombolo/core 0.46.1 → 0.48.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@crombolo/core",
3
- "version": "0.46.1",
3
+ "version": "0.48.0",
4
4
  "description": "Internal to crombolo, with no compatibility promise: use @crombolo/crombolo. Domain model, entity keys and shared contracts for crombolo. No I/O.",
5
5
  "license": "MIT",
6
6
  "author": "Crombolo",
@@ -28,6 +28,6 @@
28
28
  "access": "public"
29
29
  },
30
30
  "dependencies": {
31
- "@crombolo/common": "0.46.1"
31
+ "@crombolo/common": "0.48.0"
32
32
  }
33
33
  }
package/src/documents.ts CHANGED
@@ -84,7 +84,15 @@ export interface DocumentTreeItem {
84
84
  isTrashed: boolean
85
85
  createDate: Date
86
86
  ancestorKeys: string[]
87
- variants: Array<{ culture: string | null; name: string; state: VariantState }>
87
+ /** Public access rules apply: its own, or a page's above it. */
88
+ isProtected?: boolean
89
+ variants: Array<{
90
+ culture: string | null
91
+ name: string
92
+ state: VariantState
93
+ /** A publish is scheduled for this variant and has not run yet. */
94
+ scheduledForPublish?: boolean
95
+ }>
88
96
  }
89
97
 
90
98
  /**
package/src/index.ts CHANGED
@@ -31,6 +31,8 @@ export * from './problem-details.ts'
31
31
  export * from './property-values.ts'
32
32
  export * from './public-access.ts'
33
33
  export * from './return-urls.ts'
34
+ export * from './search-conformance.ts'
35
+ export * from './search-providers.ts'
34
36
  export * from './sections.ts'
35
37
  export * from './uploads.ts'
36
38
  export * from './users.ts'
@@ -0,0 +1,246 @@
1
+ /**
2
+ * The checks every search provider is held to (story 273), run against a
3
+ * provider by its own tests: writing, replacing and removing, word-prefix
4
+ * matching, culture, content types, access, ranking, paging and totals. It
5
+ * needs no test framework and answers a report, as the remote protocol's
6
+ * suite does. docs/search.md, "Providers".
7
+ */
8
+ import type {
9
+ BackofficeSearchEntry,
10
+ BackofficeSearchProvider,
11
+ SearchItem,
12
+ SearchKind,
13
+ SiteSearchEntry,
14
+ SiteSearchProvider,
15
+ SiteSearchText,
16
+ } from './search-providers.ts'
17
+ import { searchWords } from './search-providers.ts'
18
+
19
+ export interface SearchConformanceCheck {
20
+ name: string
21
+ ok: boolean
22
+ detail?: string
23
+ }
24
+
25
+ export interface SearchConformanceReport {
26
+ passed: boolean
27
+ checks: SearchConformanceCheck[]
28
+ }
29
+
30
+ export interface SearchConformanceOptions {
31
+ backoffice?: BackofficeSearchProvider
32
+ site?: SiteSearchProvider
33
+ /**
34
+ * Makes an item the provider may be given: a real page, media item or
35
+ * member where the provider reads its store, a made-up one where it does
36
+ * not. Pages of one call share no ancestry unless `parent` says so.
37
+ */
38
+ item(spec: {
39
+ kind: SearchKind
40
+ contentTypeAlias: string
41
+ parent?: SearchItem
42
+ }): Promise<SearchItem>
43
+ }
44
+
45
+ class Failed extends Error {}
46
+
47
+ function expect(condition: boolean, detail: string): void {
48
+ if (!condition) throw new Failed(detail)
49
+ }
50
+
51
+ const words = (...texts: string[]) => texts.flatMap(searchWords).join(' ')
52
+
53
+ const backofficeEntry = (name: string, body = '', culture = ''): BackofficeSearchEntry => ({
54
+ culture,
55
+ name: words(name),
56
+ body: words(body),
57
+ fields: { nodeName: [name], ...(body ? { bodyText: [body] } : {}) },
58
+ })
59
+
60
+ const siteText = (name: string, text: Partial<SiteSearchText> = {}): SiteSearchText => ({
61
+ name,
62
+ high: text.high ?? [],
63
+ normal: text.normal ?? [],
64
+ low: text.low ?? [],
65
+ })
66
+
67
+ const siteEntry = (
68
+ culture: string,
69
+ name: string,
70
+ text: Partial<SiteSearchText> = {},
71
+ ): SiteSearchEntry => ({ culture, text: siteText(name, text) })
72
+
73
+ /** Runs every check that applies to the providers given, each on items of its own. */
74
+ export async function searchProviderConformance(
75
+ options: SearchConformanceOptions,
76
+ ): Promise<SearchConformanceReport> {
77
+ const checks: SearchConformanceCheck[] = []
78
+ const check = async (name: string, run: () => Promise<void>) => {
79
+ try {
80
+ await run()
81
+ checks.push({ name, ok: true })
82
+ } catch (error) {
83
+ checks.push({
84
+ name,
85
+ ok: false,
86
+ detail: error instanceof Error ? error.message : String(error),
87
+ })
88
+ }
89
+ }
90
+ const { backoffice, site, item } = options
91
+
92
+ if (backoffice) {
93
+ await check('backoffice: every word of a query matches as a prefix', async () => {
94
+ const a = await item({ kind: 'document', contentTypeAlias: 'article' })
95
+ const b = await item({ kind: 'document', contentTypeAlias: 'article' })
96
+ await backoffice.write(a, [backofficeEntry('Harbour lights', 'Evening ferries')])
97
+ await backoffice.write(b, [backofficeEntry('Harbour walk')])
98
+ const ids = async (query: string) =>
99
+ (await backoffice.matches(query, { kinds: ['document'], limit: 100 })).map((m) => m.id)
100
+ expect((await ids('harb')).sort().join() === [a.id, b.id].sort().join(), 'prefix of a name')
101
+ expect((await ids('harbour ferr')).join() === String(a.id), 'every word must match')
102
+ expect((await ids('ÉVENING')).join() === String(a.id), 'folded: case and accents')
103
+ expect((await ids('submarine')).length === 0, 'a word in neither')
104
+ })
105
+ await check('backoffice: kinds and the limit narrow the matches', async () => {
106
+ const page = await item({ kind: 'document', contentTypeAlias: 'article' })
107
+ const member = await item({ kind: 'member', contentTypeAlias: 'member' })
108
+ await backoffice.write(page, [backofficeEntry('Quokka page')])
109
+ await backoffice.write(member, [backofficeEntry('Quokka member')])
110
+ const of = async (kinds: SearchKind[], limit = 100) =>
111
+ (await backoffice.matches('quokka', { kinds, limit })).map((m) => m.id)
112
+ expect((await of(['member'])).join() === String(member.id), 'members only')
113
+ expect((await of(['document'])).join() === String(page.id), 'pages only')
114
+ expect((await of(['document', 'member'], 1)).length === 1, 'at most the limit')
115
+ })
116
+ await check('backoffice: a write replaces, no entries or a removal forget', async () => {
117
+ const a = await item({ kind: 'document', contentTypeAlias: 'article' })
118
+ const b = await item({ kind: 'document', contentTypeAlias: 'article' })
119
+ await backoffice.write(a, [backofficeEntry('Wombat')])
120
+ await backoffice.write(a, [backofficeEntry('Numbat')])
121
+ await backoffice.write(b, [backofficeEntry('Wombat burrow')])
122
+ const ids = async (query: string) =>
123
+ (await backoffice.matches(query, { kinds: ['document'], limit: 100 })).map((m) => m.id)
124
+ expect(!(await ids('wombat')).includes(a.id), 'the old text is gone')
125
+ expect((await ids('numbat')).join() === String(a.id), 'the new text is found')
126
+ await backoffice.write(a, [])
127
+ expect((await ids('numbat')).length === 0, 'no entries: nothing to find')
128
+ await backoffice.remove(b)
129
+ expect((await ids('wombat')).length === 0, 'removed')
130
+ })
131
+ await check('backoffice: the dashboard gets stored fields, best first, paged', async () => {
132
+ const a = await item({ kind: 'media', contentTypeAlias: 'image' })
133
+ const b = await item({ kind: 'media', contentTypeAlias: 'image' })
134
+ await backoffice.write(a, [backofficeEntry('Pelican', 'pelican pelican')])
135
+ await backoffice.write(b, [backofficeEntry('Shore', 'a pelican')])
136
+ const all = await backoffice.query('InternalIndex', 'pelican', { skip: 0, take: 10 })
137
+ expect(all.total === 2, `total is ${all.total}, not 2`)
138
+ expect(all.items[0]?.key === a.key, 'the item named for it ranks first')
139
+ const fields = all.items[0]?.fields.find((f) => f.name === 'nodeName')
140
+ expect(fields?.values.includes('Pelican') === true, 'stored fields come back')
141
+ const second = await backoffice.query('InternalIndex', 'pelican', { skip: 1, take: 1 })
142
+ expect(
143
+ second.total === 2 && second.items.length === 1 && second.items[0]?.key === b.key,
144
+ 'skip and take page the hits, and total counts them all',
145
+ )
146
+ const members = await backoffice.query('MembersIndex', 'pelican', { skip: 0, take: 10 })
147
+ expect(members.total === 0, 'the members index holds no media')
148
+ })
149
+ await check('backoffice: stats count what each index holds', async () => {
150
+ const before = await backoffice.stats('MembersIndex')
151
+ const member = await item({ kind: 'member', contentTypeAlias: 'member' })
152
+ await backoffice.write(member, [backofficeEntry('Ada', 'one two three')])
153
+ const after = await backoffice.stats('MembersIndex')
154
+ expect(after.documentCount === before.documentCount + 1, 'one more member')
155
+ expect(after.fieldCount >= 2, 'the fields of the fullest item')
156
+ })
157
+ }
158
+
159
+ if (site) {
160
+ await check('site: culture, and an invariant page in every culture', async () => {
161
+ const both = await item({ kind: 'document', contentTypeAlias: 'article' })
162
+ const invariant = await item({ kind: 'document', contentTypeAlias: 'article' })
163
+ await site.write(both, [
164
+ siteEntry('en-us', 'Ferry timetable'),
165
+ siteEntry('da-dk', 'Færgekøreplan', { normal: ['ferry'] }),
166
+ ])
167
+ await site.write(invariant, [siteEntry('', 'Ferry news')])
168
+ const keys = async (culture: string) =>
169
+ (await site.query('ferry', { culture, skip: 0, take: 10 })).items
170
+ .map((h) => `${h.key}/${h.culture}`)
171
+ .sort()
172
+ expect(
173
+ (await keys('en-us')).join() === [`${both.key}/en-us`, `${invariant.key}/`].sort().join(),
174
+ 'English, and the invariant page',
175
+ )
176
+ expect(
177
+ (await keys('da-dk')).join() === [`${both.key}/da-dk`, `${invariant.key}/`].sort().join(),
178
+ 'Danish, and the invariant page',
179
+ )
180
+ })
181
+ await check('site: the name outranks high, normal and low text', async () => {
182
+ const low = await item({ kind: 'document', contentTypeAlias: 'article' })
183
+ const high = await item({ kind: 'document', contentTypeAlias: 'article' })
184
+ const named = await item({ kind: 'document', contentTypeAlias: 'article' })
185
+ await site.write(low, [siteEntry('', 'One', { low: ['cormorant'] })])
186
+ await site.write(high, [siteEntry('', 'Two', { high: ['cormorant'] })])
187
+ await site.write(named, [siteEntry('', 'Cormorant')])
188
+ const order = (await site.query('cormorant', { culture: '', skip: 0, take: 10 })).items.map(
189
+ (h) => h.key,
190
+ )
191
+ expect(order.join() === [named.key, high.key, low.key].join(), `ranked ${order.join()}`)
192
+ const hit = (await site.query('cormorant', { culture: '', skip: 0, take: 1 })).items[0]
193
+ expect(hit?.text.name === 'Cormorant', 'the text comes back for an excerpt')
194
+ })
195
+ await check('site: content types and paging, with the total', async () => {
196
+ const a = await item({ kind: 'document', contentTypeAlias: 'article' })
197
+ const b = await item({ kind: 'document', contentTypeAlias: 'news' })
198
+ const c = await item({ kind: 'document', contentTypeAlias: 'news' })
199
+ for (const page of [a, b, c]) await site.write(page, [siteEntry('', 'Lighthouse')])
200
+ const news = await site.query('lighthouse', {
201
+ culture: '',
202
+ types: ['news'],
203
+ skip: 0,
204
+ take: 10,
205
+ })
206
+ expect(
207
+ news.total === 2 && news.items.every((h) => h.key !== a.key),
208
+ 'only the types asked for, counted',
209
+ )
210
+ const page = await site.query('lighthouse', { culture: '', skip: 2, take: 2 })
211
+ expect(page.total === 3 && page.items.length === 1, 'skip and take, and the whole total')
212
+ })
213
+ await check('site: a page under a denied branch is left out, and counted out', async () => {
214
+ const closed = await item({ kind: 'document', contentTypeAlias: 'article' })
215
+ const inside = await item({ kind: 'document', contentTypeAlias: 'article', parent: closed })
216
+ const reopened = await item({ kind: 'document', contentTypeAlias: 'article', parent: closed })
217
+ const below = await item({ kind: 'document', contentTypeAlias: 'article', parent: reopened })
218
+ const open = await item({ kind: 'document', contentTypeAlias: 'article' })
219
+ for (const page of [inside, below, open]) await site.write(page, [siteEntry('', 'Albatross')])
220
+ const found = await site.query('albatross', {
221
+ culture: '',
222
+ access: { denied: [closed.id], allowed: [reopened.id] },
223
+ skip: 0,
224
+ take: 10,
225
+ })
226
+ const keys = found.items.map((h) => h.key).sort()
227
+ expect(keys.join() === [below.key, open.key].sort().join(), `found ${keys.join()}`)
228
+ expect(found.total === 2, `total is ${found.total}, not 2`)
229
+ })
230
+ await check('site: a write replaces, and a removal forgets', async () => {
231
+ const page = await item({ kind: 'document', contentTypeAlias: 'article' })
232
+ await site.write(page, [siteEntry('', 'Gannet')])
233
+ await site.write(page, [siteEntry('', 'Booby')])
234
+ const total = async (query: string) =>
235
+ (await site.query(query, { culture: '', skip: 0, take: 10 })).total
236
+ expect((await total('gannet')) === 0, 'the old text is gone')
237
+ expect((await total('booby')) === 1, 'the new text is found')
238
+ const before = (await site.stats()).documentCount
239
+ await site.remove(page)
240
+ expect((await total('booby')) === 0, 'removed')
241
+ expect((await site.stats()).documentCount === before - 1, 'one fewer counted')
242
+ })
243
+ }
244
+
245
+ return { passed: checks.every((c) => c.ok), checks }
246
+ }
@@ -0,0 +1,174 @@
1
+ /**
2
+ * The interfaces a search provider implements (story 273): one for backoffice
3
+ * search, over drafts, and one for site search, over published pages. The
4
+ * database providers in `@crombolo/data` are the defaults; a site replaces
5
+ * either in `crombolo.config.ts`. Crombolo still decides what an item's text
6
+ * is and who may see a result: a provider stores the documents it is given
7
+ * and finds them again. docs/search.md, "Providers".
8
+ */
9
+
10
+ /**
11
+ * Lowercase words without diacritics: letters and digits only, everything else
12
+ * a separator. The index stores text in this form, so the full-text engines on
13
+ * both dialects only ever split on spaces, and match the same way. A provider
14
+ * folds a query with it to meet what it was given.
15
+ */
16
+ export function searchWords(text: string): string[] {
17
+ return text
18
+ .normalize('NFKD')
19
+ .replace(/\p{M}+/gu, '')
20
+ .toLowerCase()
21
+ .split(/[^\p{L}\p{N}]+/u)
22
+ .filter((word) => word.length > 0)
23
+ }
24
+
25
+ /** The distinct words of a query, at most a dozen: what every result must hold by prefix. */
26
+ export function searchTerms(query: string): string[] {
27
+ return [...new Set(searchWords(query))].slice(0, 12)
28
+ }
29
+
30
+ /** What an indexed item is. */
31
+ export type SearchKind = 'document' | 'media' | 'member'
32
+
33
+ /** Umbraco's names for the two backoffice indexes, which the dashboard shows. */
34
+ export type BackofficeIndexName = 'InternalIndex' | 'MembersIndex'
35
+
36
+ /** The kinds each backoffice index covers. */
37
+ export const BACKOFFICE_INDEX_KINDS: Readonly<Record<BackofficeIndexName, readonly SearchKind[]>> =
38
+ {
39
+ InternalIndex: ['document', 'media'],
40
+ MembersIndex: ['member'],
41
+ }
42
+
43
+ /** The item a provider is given its documents for. */
44
+ export interface SearchItem {
45
+ /** The node's id, which `matches` answers with. */
46
+ id: number
47
+ key: string
48
+ kind: SearchKind
49
+ contentTypeAlias: string
50
+ /** Node ids from the root down to the item itself, for access filtering. */
51
+ path: readonly number[]
52
+ trashed: boolean
53
+ }
54
+
55
+ /** An item in one culture, '' when invariant: backoffice search's document. */
56
+ export interface BackofficeSearchEntry {
57
+ culture: string
58
+ /** Folded words: the name, and for a member its e-mail and username. */
59
+ name: string
60
+ /** Folded words of the item's draft values. */
61
+ body: string
62
+ /** The same text by property alias, as the Examine dashboard shows it. */
63
+ fields: Record<string, string[]>
64
+ }
65
+
66
+ /** A page's text in one culture, by the weight `site-search` gives each property. */
67
+ export interface SiteSearchText {
68
+ name: string
69
+ high: string[]
70
+ normal: string[]
71
+ low: string[]
72
+ }
73
+
74
+ /** A published page in one culture, '' when invariant: site search's document. */
75
+ export interface SiteSearchEntry {
76
+ culture: string
77
+ text: SiteSearchText
78
+ }
79
+
80
+ export interface SearchMatch {
81
+ id: number
82
+ score: number
83
+ }
84
+
85
+ export interface BackofficeSearchHit {
86
+ key: string
87
+ score: number
88
+ fields: Array<{ name: string; values: string[] }>
89
+ }
90
+
91
+ /**
92
+ * Protected nodes, by id, that the person searching may not open and may
93
+ * (`branchAccess`). A page is left out when the nearest of them on its path is
94
+ * denied: `openTo` says so.
95
+ */
96
+ export interface SearchAccess {
97
+ denied: readonly number[]
98
+ allowed: readonly number[]
99
+ }
100
+
101
+ export interface SiteSearchQuery {
102
+ /** Lowercase; an invariant page ('' culture) matches every culture. */
103
+ culture: string
104
+ /** Content type aliases; every type when empty or absent. */
105
+ types?: readonly string[]
106
+ access?: SearchAccess
107
+ skip: number
108
+ take: number
109
+ }
110
+
111
+ export interface SiteSearchHit {
112
+ key: string
113
+ culture: string
114
+ score: number
115
+ /** The text an excerpt is cut from. */
116
+ text: SiteSearchText
117
+ }
118
+
119
+ export interface SearchProviderHealth {
120
+ ok: boolean
121
+ detail?: string
122
+ }
123
+
124
+ export interface BackofficeSearchProvider {
125
+ /** Shown in settings and on the status check. */
126
+ readonly name: string
127
+ /** Replaces what is stored for the item; no entries means nothing to find. */
128
+ write(item: SearchItem, entries: readonly BackofficeSearchEntry[]): Promise<void>
129
+ /** The item was deleted for good. */
130
+ remove(item: Pick<SearchItem, 'id' | 'key' | 'kind'>): Promise<void>
131
+ /** Items holding every word of `query` as a word prefix, best first, at most `limit`. */
132
+ matches(
133
+ query: string,
134
+ options: { kinds: readonly SearchKind[]; limit: number },
135
+ ): Promise<SearchMatch[]>
136
+ /** The Examine dashboard's search: every stored field of each hit, best first. */
137
+ query(
138
+ index: BackofficeIndexName,
139
+ query: string,
140
+ paging: { skip: number; take: number },
141
+ ): Promise<{ total: number; items: BackofficeSearchHit[] }>
142
+ stats(index: BackofficeIndexName): Promise<{ documentCount: number; fieldCount: number }>
143
+ /** Forgets an index before it is rebuilt; absent where a rebuild overwrites in place. */
144
+ clear?(index: BackofficeIndexName): Promise<void>
145
+ health?(): Promise<SearchProviderHealth>
146
+ }
147
+
148
+ export interface SiteSearchProvider {
149
+ readonly name: string
150
+ /** Replaces what is stored for the page; no entries means it is not to be found. */
151
+ write(item: SearchItem, entries: readonly SiteSearchEntry[]): Promise<void>
152
+ remove(item: Pick<SearchItem, 'id' | 'key' | 'kind'>): Promise<void>
153
+ /** Pages holding every word as a word prefix, best first; `total` counts what may be seen. */
154
+ query(query: string, options: SiteSearchQuery): Promise<{ total: number; items: SiteSearchHit[] }>
155
+ stats(): Promise<{ documentCount: number }>
156
+ clear?(): Promise<void>
157
+ health?(): Promise<SearchProviderHealth>
158
+ }
159
+
160
+ /**
161
+ * Whether a page on `path` (root first, the page last) is open to the person
162
+ * `access` describes: the nearest protected node on it decides. Pure.
163
+ */
164
+ export function openTo(path: readonly number[], access: SearchAccess | undefined): boolean {
165
+ if (!access) return true
166
+ const denied = new Set(access.denied)
167
+ const allowed = new Set(access.allowed)
168
+ for (let i = path.length - 1; i >= 0; i--) {
169
+ const id = path[i] as number
170
+ if (denied.has(id)) return false
171
+ if (allowed.has(id)) return true
172
+ }
173
+ return true
174
+ }