@crombolo/server 0.10.0 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +11 -11
- package/src/adapters/audit.ts +186 -0
- package/src/adapters/search-index.ts +31 -0
- package/src/config.ts +14 -1
- package/src/index.ts +1 -0
- package/src/jobs.ts +74 -10
- package/src/ports.ts +6 -0
- package/src/server.ts +73 -2
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@crombolo/server",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.11.0",
|
|
4
4
|
"description": "The crombolo HTTP server: routing, configuration, media stores and background jobs.",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"author": "Crombolo",
|
|
@@ -28,16 +28,16 @@
|
|
|
28
28
|
"access": "public"
|
|
29
29
|
},
|
|
30
30
|
"dependencies": {
|
|
31
|
-
"@crombolo/api-management": "0.
|
|
32
|
-
"@crombolo/assistant": "0.
|
|
33
|
-
"@crombolo/auth": "0.
|
|
34
|
-
"@crombolo/backoffice-host": "0.
|
|
35
|
-
"@crombolo/contracts": "0.
|
|
36
|
-
"@crombolo/core": "0.
|
|
37
|
-
"@crombolo/data": "0.
|
|
38
|
-
"@crombolo/render": "0.
|
|
39
|
-
"@crombolo/schema": "0.
|
|
40
|
-
"@crombolo/transfer": "0.
|
|
31
|
+
"@crombolo/api-management": "0.11.0",
|
|
32
|
+
"@crombolo/assistant": "0.11.0",
|
|
33
|
+
"@crombolo/auth": "0.11.0",
|
|
34
|
+
"@crombolo/backoffice-host": "0.11.0",
|
|
35
|
+
"@crombolo/contracts": "0.11.0",
|
|
36
|
+
"@crombolo/core": "0.11.0",
|
|
37
|
+
"@crombolo/data": "0.11.0",
|
|
38
|
+
"@crombolo/render": "0.11.0",
|
|
39
|
+
"@crombolo/schema": "0.11.0",
|
|
40
|
+
"@crombolo/transfer": "0.11.0",
|
|
41
41
|
"@logtape/logtape": "^2.3.8"
|
|
42
42
|
}
|
|
43
43
|
}
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The audit log's ports: where the router writes, what an item's Info tab reads,
|
|
3
|
+
* and the Settings dashboard's endpoint. docs/audit-log.md.
|
|
4
|
+
*/
|
|
5
|
+
import type { AuditHistoryPort, AuditSink } from '@crombolo/api-management'
|
|
6
|
+
import {
|
|
7
|
+
AUDIT_CATEGORIES,
|
|
8
|
+
type AuditCategory,
|
|
9
|
+
type AuditEntry,
|
|
10
|
+
type AuditEntryInput,
|
|
11
|
+
AuditLog,
|
|
12
|
+
type Db,
|
|
13
|
+
} from '@crombolo/data'
|
|
14
|
+
import { logger } from '../logging.ts'
|
|
15
|
+
|
|
16
|
+
const log = logger('Audit')
|
|
17
|
+
|
|
18
|
+
/** The oldest moment the log still keeps, or undefined when it keeps everything. */
|
|
19
|
+
export function auditRetainedSince(retentionDays: number, now = new Date()): Date | undefined {
|
|
20
|
+
return retentionDays > 0 ? new Date(now.getTime() - retentionDays * 86_400_000) : undefined
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Writes an entry without ever failing what it records: the change has already
|
|
25
|
+
* been made by the time it is written, so a failure here is logged, not raised.
|
|
26
|
+
*/
|
|
27
|
+
export async function recordAudit(
|
|
28
|
+
db: Db,
|
|
29
|
+
entries: AuditEntryInput | readonly AuditEntryInput[],
|
|
30
|
+
): Promise<void> {
|
|
31
|
+
try {
|
|
32
|
+
await new AuditLog(db).record(entries)
|
|
33
|
+
} catch (error) {
|
|
34
|
+
log.error('Could not write to the audit log: {error}', { error })
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
export function createAuditSink(db: Db): AuditSink {
|
|
39
|
+
const audit = new AuditLog(db)
|
|
40
|
+
return {
|
|
41
|
+
async describe(entityType, key) {
|
|
42
|
+
try {
|
|
43
|
+
return await audit.describe(entityType, key)
|
|
44
|
+
} catch (error) {
|
|
45
|
+
log.warning('Could not name {entityType} {key} for the audit log: {error}', {
|
|
46
|
+
entityType,
|
|
47
|
+
key,
|
|
48
|
+
error,
|
|
49
|
+
})
|
|
50
|
+
return { entityType, entityKey: key, entityName: null }
|
|
51
|
+
}
|
|
52
|
+
},
|
|
53
|
+
record: (entries) =>
|
|
54
|
+
recordAudit(
|
|
55
|
+
db,
|
|
56
|
+
entries.map((entry) => ({ ...entry, source: 'request' as const })),
|
|
57
|
+
),
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
export function createAuditHistory(db: Db, retentionDays: number): AuditHistoryPort {
|
|
62
|
+
const audit = new AuditLog(db)
|
|
63
|
+
let started: Date | undefined
|
|
64
|
+
return {
|
|
65
|
+
async forItem(key) {
|
|
66
|
+
const { items } = await audit.query({ entityKey: key }, { skip: 0, take: 10_000 })
|
|
67
|
+
return items
|
|
68
|
+
},
|
|
69
|
+
async startedAt() {
|
|
70
|
+
started ??= await audit.startedAt()
|
|
71
|
+
return started
|
|
72
|
+
},
|
|
73
|
+
retainedSince: () => auditRetainedSince(retentionDays),
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function toResponse(entry: AuditEntry) {
|
|
78
|
+
return {
|
|
79
|
+
id: entry.id,
|
|
80
|
+
occurredAt: entry.occurredAt.toISOString(),
|
|
81
|
+
category: entry.category,
|
|
82
|
+
action: entry.action,
|
|
83
|
+
source: entry.source,
|
|
84
|
+
user: entry.userKey || entry.userName ? { key: entry.userKey, name: entry.userName } : null,
|
|
85
|
+
item: entry.entityType
|
|
86
|
+
? { type: entry.entityType, key: entry.entityKey, name: entry.entityName }
|
|
87
|
+
: null,
|
|
88
|
+
ipAddress: entry.ipAddress,
|
|
89
|
+
forwardedFor: entry.forwardedFor,
|
|
90
|
+
detail: entry.detail,
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
function dateParam(value: string | null): Date | undefined | null {
|
|
95
|
+
if (!value) return undefined
|
|
96
|
+
const parsed = new Date(value)
|
|
97
|
+
return Number.isNaN(parsed.getTime()) ? null : parsed
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const badRequest = (detail: string) =>
|
|
101
|
+
Response.json({ title: 'Invalid request', status: 400, detail }, { status: 400 })
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* The Settings dashboard's reads: `GET /` lists entries, newest first, filtered
|
|
105
|
+
* by user, item, category, action, date and text; `GET /users` names everyone in
|
|
106
|
+
* the log. Nothing here writes, so an entry cannot be changed through it.
|
|
107
|
+
*/
|
|
108
|
+
export function createAuditHost(db: Db, retentionDays: number) {
|
|
109
|
+
const audit = new AuditLog(db)
|
|
110
|
+
return {
|
|
111
|
+
async handle(request: Request, pathname: string): Promise<Response | undefined> {
|
|
112
|
+
if (pathname !== '' && pathname !== '/' && pathname !== '/users') return undefined
|
|
113
|
+
if (request.method !== 'GET')
|
|
114
|
+
return new Response('Method Not Allowed', { status: 405, headers: { allow: 'GET' } })
|
|
115
|
+
if (pathname === '/users') return Response.json({ items: await audit.users() })
|
|
116
|
+
|
|
117
|
+
const search = new URL(request.url).searchParams
|
|
118
|
+
const skip = Number(search.get('skip') ?? 0)
|
|
119
|
+
const take = Number(search.get('take') ?? 50)
|
|
120
|
+
if (!Number.isInteger(skip) || skip < 0 || !Number.isInteger(take) || take < 1 || take > 500)
|
|
121
|
+
return badRequest('skip must be 0 or more, and take between 1 and 500.')
|
|
122
|
+
const category = search.get('category') ?? undefined
|
|
123
|
+
if (category && !AUDIT_CATEGORIES.includes(category as AuditCategory))
|
|
124
|
+
return badRequest(`There is no category "${category}".`)
|
|
125
|
+
const since = dateParam(search.get('since'))
|
|
126
|
+
const until = dateParam(search.get('until'))
|
|
127
|
+
if (since === null || until === null) return badRequest('since and until must be dates.')
|
|
128
|
+
const result = await audit.query(
|
|
129
|
+
{
|
|
130
|
+
userKey: search.get('user') ?? undefined,
|
|
131
|
+
entityKey: search.get('item') ?? undefined,
|
|
132
|
+
category: category as AuditCategory | undefined,
|
|
133
|
+
action: search.get('action') ?? undefined,
|
|
134
|
+
since,
|
|
135
|
+
until,
|
|
136
|
+
text: search.get('text') ?? undefined,
|
|
137
|
+
},
|
|
138
|
+
{ skip, take },
|
|
139
|
+
)
|
|
140
|
+
return Response.json({
|
|
141
|
+
total: result.total,
|
|
142
|
+
retentionDays,
|
|
143
|
+
items: result.items.map(toResponse),
|
|
144
|
+
})
|
|
145
|
+
},
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
const VERB: Record<string, string> = {
|
|
150
|
+
POST: 'Create',
|
|
151
|
+
PUT: 'Save',
|
|
152
|
+
PATCH: 'Save',
|
|
153
|
+
DELETE: 'Delete',
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/** Crombolo's own endpoints that change something, by their first segment. */
|
|
157
|
+
const PLUGIN_AREAS: Record<string, { category: AuditCategory; entityType: string }> = {
|
|
158
|
+
'schema-import': { category: 'schema', entityType: 'Schema' },
|
|
159
|
+
git: { category: 'schema', entityType: 'Schema' },
|
|
160
|
+
seo: { category: 'settings', entityType: 'SeoSettings' },
|
|
161
|
+
bundles: { category: 'system', entityType: 'Bundle' },
|
|
162
|
+
forms: { category: 'content', entityType: 'Form' },
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
const PLUGIN_ACTIONS: Record<string, string> = {
|
|
166
|
+
'schema-import': 'Import',
|
|
167
|
+
'git/commit': 'Commit',
|
|
168
|
+
'bundles/install': 'Install',
|
|
169
|
+
'bundles/uninstall': 'Uninstall',
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
/** The entry for a write to `<plugin>/api/<subpath>`: the area it is in, and what it did there. */
|
|
173
|
+
export function pluginAuditPlan(
|
|
174
|
+
method: string,
|
|
175
|
+
subpath: string,
|
|
176
|
+
): Pick<AuditEntryInput, 'category' | 'action' | 'entityType' | 'entityKey' | 'detail'> {
|
|
177
|
+
const [area = '', ...rest] = subpath.split('/')
|
|
178
|
+
const known = PLUGIN_AREAS[area]
|
|
179
|
+
return {
|
|
180
|
+
category: known?.category ?? 'settings',
|
|
181
|
+
entityType: known?.entityType ?? area,
|
|
182
|
+
action: PLUGIN_ACTIONS[subpath] ?? VERB[method] ?? method,
|
|
183
|
+
entityKey: rest.length > 0 ? rest.join('/') : null,
|
|
184
|
+
detail: `${method} ${subpath}`,
|
|
185
|
+
}
|
|
186
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/** The Examine dashboard's port over the search index. docs/search.md. */
|
|
2
|
+
import type { SearchIndexInfo, SearchIndexPort } from '@crombolo/api-management'
|
|
3
|
+
import { type Db, SEARCH_INDEXES, SearchIndex, type SearchIndexDefinition } from '@crombolo/data'
|
|
4
|
+
|
|
5
|
+
export function createSearchIndexPort(db: Db, onQueued?: () => void): SearchIndexPort {
|
|
6
|
+
const search = new SearchIndex(db)
|
|
7
|
+
const info = async (index: SearchIndexDefinition): Promise<SearchIndexInfo> => ({
|
|
8
|
+
name: index.name,
|
|
9
|
+
searcherName: index.searcherName,
|
|
10
|
+
...(await search.stats(index)),
|
|
11
|
+
})
|
|
12
|
+
return {
|
|
13
|
+
indexes: () => Promise.all(SEARCH_INDEXES.map(info)),
|
|
14
|
+
async index(name) {
|
|
15
|
+
const index = SearchIndex.definition(name)
|
|
16
|
+
return index ? info(index) : undefined
|
|
17
|
+
},
|
|
18
|
+
async rebuild(name) {
|
|
19
|
+
const index = SearchIndex.definition(name)
|
|
20
|
+
if (!index) return false
|
|
21
|
+
await search.rebuild(index)
|
|
22
|
+
onQueued?.()
|
|
23
|
+
return true
|
|
24
|
+
},
|
|
25
|
+
searchers: () => SEARCH_INDEXES.map((index) => index.searcherName),
|
|
26
|
+
async query(searcherName, term, paging) {
|
|
27
|
+
const index = SearchIndex.bySearcher(searcherName)
|
|
28
|
+
return index ? search.query(index, term, paging) : undefined
|
|
29
|
+
},
|
|
30
|
+
}
|
|
31
|
+
}
|
package/src/config.ts
CHANGED
|
@@ -161,6 +161,8 @@ export interface CromboloConfig {
|
|
|
161
161
|
keepAllVersionsNewerThanDays: number
|
|
162
162
|
keepLatestVersionPerDayForDays: number
|
|
163
163
|
}
|
|
164
|
+
/** How many days the audit log keeps an entry; 0 keeps them forever. docs/audit-log.md. */
|
|
165
|
+
auditRetentionDays: number
|
|
164
166
|
/** Where uploaded media files live; served at `/media/`. */
|
|
165
167
|
mediaDir: string
|
|
166
168
|
/**
|
|
@@ -317,6 +319,7 @@ export const DEFAULTS = {
|
|
|
317
319
|
npmRegistry: 'https://registry.npmjs.org',
|
|
318
320
|
keepAllVersionsNewerThanDays: 7,
|
|
319
321
|
keepLatestVersionPerDayForDays: 90,
|
|
322
|
+
auditRetentionDays: 365,
|
|
320
323
|
memberRegistrationType: 'Member',
|
|
321
324
|
memberSessionMinutes: 60 * 24 * 14,
|
|
322
325
|
maxFailedPasswordAttempts: 5,
|
|
@@ -440,6 +443,10 @@ export function loadConfig(
|
|
|
440
443
|
DEFAULTS.keepLatestVersionPerDayForDays,
|
|
441
444
|
),
|
|
442
445
|
},
|
|
446
|
+
auditRetentionDays: nonNegative(
|
|
447
|
+
Bun.env.CROMBOLO_AUDIT_RETENTION_DAYS,
|
|
448
|
+
DEFAULTS.auditRetentionDays,
|
|
449
|
+
),
|
|
443
450
|
mediaDir: Bun.env.CROMBOLO_MEDIA_DIR ?? DEFAULTS.mediaDir,
|
|
444
451
|
mediaStore: undefined,
|
|
445
452
|
logsDir: Bun.env.CROMBOLO_LOGS_DIR ?? DEFAULTS.logsDir,
|
|
@@ -518,6 +525,12 @@ export function loadConfig(
|
|
|
518
525
|
}
|
|
519
526
|
|
|
520
527
|
/** `true`/`false` from the environment; undefined when unset, so a default applies. */
|
|
528
|
+
/** A whole number of at least 0; anything else falls back, rather than ageing the log out by accident. */
|
|
529
|
+
function nonNegative(value: string | undefined, fallback: number): number {
|
|
530
|
+
const parsed = value === undefined ? Number.NaN : Number(value)
|
|
531
|
+
return Number.isInteger(parsed) && parsed >= 0 ? parsed : fallback
|
|
532
|
+
}
|
|
533
|
+
|
|
521
534
|
function envFlag(value: string | undefined): boolean | undefined {
|
|
522
535
|
return value === undefined ? undefined : value !== 'false'
|
|
523
536
|
}
|
|
@@ -553,4 +566,4 @@ export function describeDatabase(
|
|
|
553
566
|
}
|
|
554
567
|
}
|
|
555
568
|
|
|
556
|
-
export const VERSION = '0.
|
|
569
|
+
export const VERSION = '0.11.0'
|
package/src/index.ts
CHANGED
package/src/jobs.ts
CHANGED
|
@@ -1,13 +1,21 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Background jobs, as Umbraco runs them: scheduled publishing every minute and
|
|
3
|
-
* version cleanup every hour when it is enabled
|
|
3
|
+
* version cleanup every hour when it is enabled, plus the search indexer
|
|
4
|
+
* draining its queue and the audit log ageing out old entries hourly. Each run can also be called directly, which is
|
|
4
5
|
* how tests drive them. On a multi-node Postgres site every node runs them;
|
|
5
|
-
* each schedule is claimed atomically, so it runs once.
|
|
6
|
+
* each schedule and queued item is claimed atomically, so it runs once.
|
|
6
7
|
*/
|
|
7
8
|
|
|
8
9
|
import type { NodeSchemaState } from '@crombolo/data'
|
|
9
|
-
import {
|
|
10
|
+
import {
|
|
11
|
+
AuditLog,
|
|
12
|
+
type Db,
|
|
13
|
+
DocumentRepository,
|
|
14
|
+
PublishBlockedError,
|
|
15
|
+
SearchIndex,
|
|
16
|
+
} from '@crombolo/data'
|
|
10
17
|
import type { PublishedCache } from '@crombolo/render'
|
|
18
|
+
import { auditRetainedSince, recordAudit } from './adapters/audit.ts'
|
|
11
19
|
import type { WorkflowRunner, WorkflowRunReport } from './form-workflows.ts'
|
|
12
20
|
import { logger } from './logging.ts'
|
|
13
21
|
import type { MediaFileStore } from './media-files.ts'
|
|
@@ -33,6 +41,10 @@ export interface JobsOptions {
|
|
|
33
41
|
*/
|
|
34
42
|
formWorkflows?: WorkflowRunner
|
|
35
43
|
formWorkflowsMs?: number
|
|
44
|
+
/** How often the search indexer looks for queued items; a write here also starts it at once. */
|
|
45
|
+
searchIndexingMs?: number
|
|
46
|
+
/** Days the audit log keeps an entry; 0 or absent keeps them forever. */
|
|
47
|
+
auditRetentionDays?: number
|
|
36
48
|
}
|
|
37
49
|
|
|
38
50
|
export interface ScheduledRun {
|
|
@@ -45,6 +57,12 @@ export interface BackgroundJobs {
|
|
|
45
57
|
runScheduledPublishing(now?: Date): Promise<ScheduledRun>
|
|
46
58
|
/** Runs the form workflows that are due; a no-op when none are configured. */
|
|
47
59
|
runFormWorkflows(now?: Date): Promise<WorkflowRunReport>
|
|
60
|
+
/** Indexes every queued item, and returns how many. docs/search.md. */
|
|
61
|
+
runSearchIndexing(): Promise<number>
|
|
62
|
+
/** Something was queued here: index it shortly, rather than at the next tick. */
|
|
63
|
+
kickSearchIndexing(): void
|
|
64
|
+
/** Deletes audit entries older than the retention period, and returns how many. */
|
|
65
|
+
runAuditRetention(now?: Date): Promise<number>
|
|
48
66
|
/** Runs whether or not the hourly job is enabled; the policy decides what goes. */
|
|
49
67
|
runVersionCleanup(now?: Date): Promise<{ nodes: number; versionsDeleted: number }>
|
|
50
68
|
start(): void
|
|
@@ -60,6 +78,9 @@ export function createBackgroundJobs(options: JobsOptions): BackgroundJobs {
|
|
|
60
78
|
})
|
|
61
79
|
const timers: Array<ReturnType<typeof setInterval>> = []
|
|
62
80
|
let running = false
|
|
81
|
+
const search = new SearchIndex(options.db)
|
|
82
|
+
let indexing: Promise<number> | undefined
|
|
83
|
+
let kicked: ReturnType<typeof setTimeout> | undefined
|
|
63
84
|
|
|
64
85
|
async function runScheduledPublishing(now = new Date()): Promise<ScheduledRun> {
|
|
65
86
|
const run: ScheduledRun = { published: [], unpublished: [], failed: [] }
|
|
@@ -67,13 +88,20 @@ export function createBackgroundJobs(options: JobsOptions): BackgroundJobs {
|
|
|
67
88
|
if (!(await documents.claimSchedule(due.id))) continue
|
|
68
89
|
const cultures = due.culture ? [due.culture] : null
|
|
69
90
|
try {
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
91
|
+
const release = due.action === 'Release'
|
|
92
|
+
const changed = release
|
|
93
|
+
? await documents.publish(due.key, cultures)
|
|
94
|
+
: await documents.unpublish(due.key, cultures)
|
|
95
|
+
;(release ? run.published : run.unpublished).push(due.key)
|
|
96
|
+
await recordAudit(options.db, {
|
|
97
|
+
category: 'content',
|
|
98
|
+
action: release ? 'Publish' : 'Unpublish',
|
|
99
|
+
source: 'job',
|
|
100
|
+
entityType: 'Document',
|
|
101
|
+
entityKey: due.key,
|
|
102
|
+
entityName: changed?.variants[0]?.name ?? null,
|
|
103
|
+
detail: `Scheduled${due.culture ? ` (${due.culture})` : ''}`,
|
|
104
|
+
})
|
|
77
105
|
options.onDocumentChanged?.(due.key)
|
|
78
106
|
} catch (error) {
|
|
79
107
|
if (!(error instanceof PublishBlockedError)) throw error
|
|
@@ -84,6 +112,11 @@ export function createBackgroundJobs(options: JobsOptions): BackgroundJobs {
|
|
|
84
112
|
return run
|
|
85
113
|
}
|
|
86
114
|
|
|
115
|
+
async function runAuditRetention(now = new Date()): Promise<number> {
|
|
116
|
+
const before = auditRetainedSince(options.auditRetentionDays ?? 0, now)
|
|
117
|
+
return before ? new AuditLog(options.db).prune(before) : 0
|
|
118
|
+
}
|
|
119
|
+
|
|
87
120
|
async function runVersionCleanup(now = new Date()) {
|
|
88
121
|
return documents.cleanupVersions(now, options.versionCleanup)
|
|
89
122
|
}
|
|
@@ -99,6 +132,20 @@ export function createBackgroundJobs(options: JobsOptions): BackgroundJobs {
|
|
|
99
132
|
)
|
|
100
133
|
}
|
|
101
134
|
|
|
135
|
+
// Its own guard rather than `guarded`: indexing a big rebuild must not hold up
|
|
136
|
+
// a scheduled publish, and a kick while it runs joins the run in progress.
|
|
137
|
+
function runSearchIndexing(): Promise<number> {
|
|
138
|
+
indexing ??= search.indexAll().finally(() => {
|
|
139
|
+
indexing = undefined
|
|
140
|
+
})
|
|
141
|
+
return indexing
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
const indexQuietly = () =>
|
|
145
|
+
runSearchIndexing().catch((error: unknown) => {
|
|
146
|
+
logger('jobs').error('Search indexing failed', { error })
|
|
147
|
+
})
|
|
148
|
+
|
|
102
149
|
const guarded = (job: () => Promise<unknown>) => async () => {
|
|
103
150
|
if (running) return
|
|
104
151
|
running = true
|
|
@@ -120,18 +167,35 @@ export function createBackgroundJobs(options: JobsOptions): BackgroundJobs {
|
|
|
120
167
|
return {
|
|
121
168
|
runScheduledPublishing,
|
|
122
169
|
runVersionCleanup,
|
|
170
|
+
runAuditRetention,
|
|
123
171
|
runFormWorkflows,
|
|
172
|
+
runSearchIndexing,
|
|
173
|
+
kickSearchIndexing() {
|
|
174
|
+
if (timers.length === 0 || kicked) return
|
|
175
|
+
kicked = setTimeout(() => {
|
|
176
|
+
kicked = undefined
|
|
177
|
+
void indexQuietly()
|
|
178
|
+
}, 250)
|
|
179
|
+
kicked.unref()
|
|
180
|
+
},
|
|
124
181
|
start() {
|
|
125
182
|
every(options.scheduledPublishingMs ?? 60_000, runScheduledPublishing)
|
|
183
|
+
// The queue is shared, so this also picks up what other nodes wrote.
|
|
184
|
+
const searchTimer = setInterval(indexQuietly, options.searchIndexingMs ?? 5_000)
|
|
185
|
+
searchTimer.unref()
|
|
186
|
+
timers.push(searchTimer)
|
|
126
187
|
// More often than the other jobs: somebody waiting on a form notification
|
|
127
188
|
// notices a minute, and a queue row is cheap to look for.
|
|
128
189
|
if (options.formWorkflows) every(options.formWorkflowsMs ?? 15_000, runFormWorkflows)
|
|
129
190
|
if (options.versionCleanup.enabled) every(60 * 60_000, runVersionCleanup)
|
|
191
|
+
if (options.auditRetentionDays) every(60 * 60_000, runAuditRetention)
|
|
130
192
|
const files = options.mediaFiles
|
|
131
193
|
if (files) every(60 * 60_000, () => files.cleanupExpired())
|
|
132
194
|
},
|
|
133
195
|
stop() {
|
|
134
196
|
for (const timer of timers.splice(0)) clearInterval(timer)
|
|
197
|
+
if (kicked) clearTimeout(kicked)
|
|
198
|
+
kicked = undefined
|
|
135
199
|
},
|
|
136
200
|
started() {
|
|
137
201
|
return timers.length > 0
|
package/src/ports.ts
CHANGED
|
@@ -35,6 +35,7 @@ import {
|
|
|
35
35
|
} from '@crombolo/data'
|
|
36
36
|
import type { PublishedCache } from '@crombolo/render'
|
|
37
37
|
import { createNodeLookup } from './access.ts'
|
|
38
|
+
import { createAuditHistory } from './adapters/audit.ts'
|
|
38
39
|
import { createBlueprintPort } from './adapters/blueprints.ts'
|
|
39
40
|
import { createPackagePort } from './adapters/bundles.ts'
|
|
40
41
|
import {
|
|
@@ -58,6 +59,7 @@ import {
|
|
|
58
59
|
type SchemaFiles,
|
|
59
60
|
type SchemaFileWriter,
|
|
60
61
|
} from './adapters/schema-files.ts'
|
|
62
|
+
import { createSearchIndexPort } from './adapters/search-index.ts'
|
|
61
63
|
import { createTemplateFileStore } from './adapters/template-files.ts'
|
|
62
64
|
import { createTemplateQueryPort } from './adapters/template-query.ts'
|
|
63
65
|
import { avatarUrls, createUserPorts } from './adapters/users.ts'
|
|
@@ -374,6 +376,8 @@ export interface DepsOptions {
|
|
|
374
376
|
* tells the other nodes to look; without it they keep the view they imported.
|
|
375
377
|
*/
|
|
376
378
|
onViewsChange?: (alias: string) => void | Promise<void>
|
|
379
|
+
/** Items were queued for the search index; the indexer can start on them now. */
|
|
380
|
+
onSearchQueued?: () => void
|
|
377
381
|
}
|
|
378
382
|
|
|
379
383
|
function createTemporaryFilePort(files: MediaFileStore): TemporaryFilePort {
|
|
@@ -514,12 +518,14 @@ export function createDeps(options: DepsOptions): ManagementApiDeps {
|
|
|
514
518
|
stylesheets: deps.stylesheets,
|
|
515
519
|
scripts: deps.scripts,
|
|
516
520
|
})
|
|
521
|
+
deps.auditHistory = createAuditHistory(options.db, options.config.auditRetentionDays)
|
|
517
522
|
if (options.cache) {
|
|
518
523
|
const cache = options.cache
|
|
519
524
|
deps.templateQuery = createTemplateQueryPort({
|
|
520
525
|
cache,
|
|
521
526
|
types: new ContentTypeRepository(options.db),
|
|
522
527
|
})
|
|
528
|
+
deps.searchIndex = createSearchIndexPort(options.db, options.onSearchQueued)
|
|
523
529
|
deps.publishedCache = {
|
|
524
530
|
reload: () => cache.invalidate(),
|
|
525
531
|
rebuild: async () => {
|
package/src/server.ts
CHANGED
|
@@ -7,7 +7,12 @@
|
|
|
7
7
|
*/
|
|
8
8
|
|
|
9
9
|
import { join } from 'node:path'
|
|
10
|
-
import {
|
|
10
|
+
import {
|
|
11
|
+
type ClientAddress,
|
|
12
|
+
createManagementApiRouter,
|
|
13
|
+
hasSection,
|
|
14
|
+
type Principal,
|
|
15
|
+
} from '@crombolo/api-management'
|
|
11
16
|
import {
|
|
12
17
|
AuthService,
|
|
13
18
|
AuthStore,
|
|
@@ -65,6 +70,7 @@ import {
|
|
|
65
70
|
import type { SyncReport } from '@crombolo/schema'
|
|
66
71
|
import type { Server } from 'bun'
|
|
67
72
|
import { createNodeLookup, loadAccess } from './access.ts'
|
|
73
|
+
import { createAuditHost, createAuditSink, pluginAuditPlan, recordAudit } from './adapters/audit.ts'
|
|
68
74
|
import { readCreatedBundlePackage, writeCreatedBundlePackage } from './adapters/bundles.ts'
|
|
69
75
|
import { createPublishedContentSource, loadDraftNode } from './adapters/documents.ts'
|
|
70
76
|
import { createSchemaProposals } from './adapters/schema-proposals.ts'
|
|
@@ -270,10 +276,30 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
|
|
|
270
276
|
const auth = new AuthService(new AuthStore(db))
|
|
271
277
|
const cookies = { siteName: config.siteName, securePrefix: config.secureCookies }
|
|
272
278
|
const names = cookieNames(cookies)
|
|
279
|
+
// The socket's peer for each request in flight, which only `Bun.serve` knows;
|
|
280
|
+
// a request that came some other way (a test, the assistant) has none.
|
|
281
|
+
const peers = new WeakMap<Request, string>()
|
|
282
|
+
const clientAddress = (request: Request): ClientAddress => ({
|
|
283
|
+
ipAddress: peers.get(request) ?? null,
|
|
284
|
+
forwardedFor: request.headers.get('x-forwarded-for'),
|
|
285
|
+
})
|
|
273
286
|
const authRoutes = createAuthRoutes({
|
|
274
287
|
backOfficePath: paths.backOfficePath,
|
|
275
288
|
service: auth,
|
|
276
289
|
cookies,
|
|
290
|
+
onAudit: (event, request) =>
|
|
291
|
+
recordAudit(db, {
|
|
292
|
+
category: 'sign-in',
|
|
293
|
+
action: event.action,
|
|
294
|
+
source: 'request',
|
|
295
|
+
userKey: event.user?.key ?? null,
|
|
296
|
+
userName: event.user?.userName ?? event.login ?? null,
|
|
297
|
+
entityType: event.user ? 'User' : null,
|
|
298
|
+
entityKey: event.user?.key ?? null,
|
|
299
|
+
entityName: event.user?.userName ?? null,
|
|
300
|
+
detail: event.detail ?? null,
|
|
301
|
+
...clientAddress(request),
|
|
302
|
+
}),
|
|
277
303
|
})
|
|
278
304
|
|
|
279
305
|
/**
|
|
@@ -535,6 +561,7 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
|
|
|
535
561
|
schema,
|
|
536
562
|
mediaFiles,
|
|
537
563
|
email,
|
|
564
|
+
onSearchQueued: () => jobs.kickSearchIndexing(),
|
|
538
565
|
// This node looks again at once; the others are told to. The write has
|
|
539
566
|
// already landed wherever `componentsDir` points, so there is nothing to
|
|
540
567
|
// distribute — only the fact that it happened.
|
|
@@ -557,11 +584,15 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
|
|
|
557
584
|
authenticate,
|
|
558
585
|
nodes: createNodeLookup(db),
|
|
559
586
|
allowLocalLogin: config.allowLocalLogin,
|
|
587
|
+
audit: createAuditSink(db),
|
|
588
|
+
clientAddress,
|
|
560
589
|
onNotImplemented: config.development
|
|
561
590
|
? (operation, request) => notImplemented.record(operation, request)
|
|
562
591
|
: undefined,
|
|
563
592
|
onServerEvent: (event) => {
|
|
564
593
|
events.broadcast('notify', [event])
|
|
594
|
+
// Saves, deletes and moves all announce themselves; whatever they queued gets indexed now.
|
|
595
|
+
jobs.kickSearchIndexing()
|
|
565
596
|
// An open preview of the document reloads when it changes.
|
|
566
597
|
if (event.eventSource === 'Umbraco:CMS:Document' && event.eventType !== 'Deleted')
|
|
567
598
|
previewHub.broadcast('refreshed', [event.key])
|
|
@@ -606,6 +637,7 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
|
|
|
606
637
|
nodeState: schema.nodeState,
|
|
607
638
|
nodeId: schema.nodeId,
|
|
608
639
|
versionCleanup: config.versionCleanup,
|
|
640
|
+
auditRetentionDays: config.auditRetentionDays,
|
|
609
641
|
mediaFiles,
|
|
610
642
|
// The workflow queue drains here rather than in the request: a submission is
|
|
611
643
|
// stored first, and an e-mail that cannot be sent yet must not cost it.
|
|
@@ -680,6 +712,9 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
|
|
|
680
712
|
|
|
681
713
|
// Imported only when configured, like the assistant: an unconfigured site has no
|
|
682
714
|
// git route and no credentials anywhere near it.
|
|
715
|
+
const auditPath = `${paths.pluginPath}/api/audit`
|
|
716
|
+
const auditHost = createAuditHost(db, config.auditRetentionDays)
|
|
717
|
+
|
|
683
718
|
const gitPath = `${paths.pluginPath}/api/git`
|
|
684
719
|
const git =
|
|
685
720
|
config.git && servesBackOffice
|
|
@@ -714,7 +749,10 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
|
|
|
714
749
|
// there — they are not routes at all, and 404 like any other unknown path.
|
|
715
750
|
if (servesBackOffice) {
|
|
716
751
|
const response = await serveBackOffice(request, url)
|
|
717
|
-
if (response)
|
|
752
|
+
if (response) {
|
|
753
|
+
if (response.ok) await auditPluginWrite(request, pathname)
|
|
754
|
+
return response
|
|
755
|
+
}
|
|
718
756
|
}
|
|
719
757
|
|
|
720
758
|
if (pathname.startsWith(`${MEMBER_ROUTE_PREFIX}/`)) {
|
|
@@ -749,6 +787,27 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
|
|
|
749
787
|
return renderSite(request, pathname, url.host)
|
|
750
788
|
}
|
|
751
789
|
|
|
790
|
+
/**
|
|
791
|
+
* The audit entry for a successful write to one of Crombolo's own endpoints,
|
|
792
|
+
* which the contract router does not see. The assistant's are left out: what it
|
|
793
|
+
* changes goes through the router, and is recorded there. docs/audit-log.md.
|
|
794
|
+
*/
|
|
795
|
+
async function auditPluginWrite(request: Request, pathname: string): Promise<void> {
|
|
796
|
+
const prefix = `${paths.pluginPath}/api/`
|
|
797
|
+
if (request.method === 'GET' || request.method === 'HEAD') return
|
|
798
|
+
if (!pathname.startsWith(prefix) || pathname.startsWith(`${assistantPath}/`)) return
|
|
799
|
+
const principal = await authenticate(request)
|
|
800
|
+
if (!principal) return
|
|
801
|
+
const plan = pluginAuditPlan(request.method, pathname.slice(prefix.length))
|
|
802
|
+
await recordAudit(db, {
|
|
803
|
+
...plan,
|
|
804
|
+
source: 'request',
|
|
805
|
+
userKey: principal.id,
|
|
806
|
+
userName: principal.name,
|
|
807
|
+
...clientAddress(request),
|
|
808
|
+
})
|
|
809
|
+
}
|
|
810
|
+
|
|
752
811
|
/**
|
|
753
812
|
* The backoffice, the Management API and everything that serves an editor.
|
|
754
813
|
*
|
|
@@ -1166,6 +1225,16 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
|
|
|
1166
1225
|
// and no explanation. The findings name documents and the properties that are
|
|
1167
1226
|
// unfilled: that is the Settings dashboard's material, and the dashboard's own
|
|
1168
1227
|
// manifest is already `Umb.Condition.SectionAlias` on Settings.
|
|
1228
|
+
// Non-contract, Settings: the audit log dashboard's reads. Settings, as the
|
|
1229
|
+
// dashboard is: it names who signed in from where, and every change made.
|
|
1230
|
+
if (pathname === auditPath || pathname.startsWith(`${auditPath}/`)) {
|
|
1231
|
+
const principal = await authenticate(request)
|
|
1232
|
+
if (!principal) return new Response('Unauthorized', { status: 401 })
|
|
1233
|
+
if (!hasSection(principal, 'settings')) return new Response('Forbidden', { status: 403 })
|
|
1234
|
+
const response = await auditHost.handle(request, pathname.slice(auditPath.length))
|
|
1235
|
+
if (response) return response
|
|
1236
|
+
}
|
|
1237
|
+
|
|
1169
1238
|
if (pathname === `${paths.pluginPath}/api/change-report`) {
|
|
1170
1239
|
const principal = await authenticate(request)
|
|
1171
1240
|
if (!principal) return new Response('Unauthorized', { status: 401 })
|
|
@@ -1605,6 +1674,8 @@ export async function createServer(config: CromboloConfig = loadConfig()): Promi
|
|
|
1605
1674
|
port: config.port,
|
|
1606
1675
|
maxRequestBodySize: config.maxRequestBodyBytes,
|
|
1607
1676
|
fetch: diagnostics.wrap(async (request, bunServer: Server<HubSocketData>) => {
|
|
1677
|
+
const peer = bunServer.requestIP(request)?.address
|
|
1678
|
+
if (peer) peers.set(request, peer)
|
|
1608
1679
|
const pathname = new URL(request.url).pathname
|
|
1609
1680
|
if (pathname === hubPath || pathname === previewHubPath) {
|
|
1610
1681
|
const principal = await authenticate(request)
|