@better-forge/config-as-code 0.1.0 → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +10 -10
- package/package.json +51 -51
- package/src/document.ts +199 -199
- package/src/engine.ts +258 -258
- package/src/index.ts +16 -16
- package/src/keys.ts +64 -64
- package/src/routes.ts +143 -143
- package/src/yaml.ts +64 -64
package/src/engine.ts
CHANGED
|
@@ -1,258 +1,258 @@
|
|
|
1
|
-
import { clip, ConfigInvalidError, createVaultExporter, createVaultResolver, forbiddenNameAt, isObject, Problems, type Json, type VaultRows } from './document.js'
|
|
2
|
-
|
|
3
|
-
export type ChangeAction = 'create' | 'update' | 'delete' | 'unchanged'
|
|
4
|
-
|
|
5
|
-
export interface ConfigChange {
|
|
6
|
-
kind: string
|
|
7
|
-
/** The key of the object the document describes. */
|
|
8
|
-
key?: string
|
|
9
|
-
action: ChangeAction
|
|
10
|
-
/** For an update: which settings differ. Never their values, which may be secrets. */
|
|
11
|
-
fields?: string[]
|
|
12
|
-
}
|
|
13
|
-
|
|
14
|
-
export interface ImportResult {
|
|
15
|
-
dryRun: boolean
|
|
16
|
-
/** `delete` is present only for an import with `prune`. */
|
|
17
|
-
summary: Record<'create' | 'update' | 'unchanged', number> & { delete?: number }
|
|
18
|
-
changes: ConfigChange[]
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
/** What one change did, for the audit log. */
|
|
22
|
-
export interface AuditEntry { action: 'create' | 'update' | 'delete'; entity: string; id: number | string; name: string; diff: Json }
|
|
23
|
-
|
|
24
|
-
export interface ImportOptions<Actor> {
|
|
25
|
-
dryRun: boolean
|
|
26
|
-
actor: Actor
|
|
27
|
-
/** Set when the request is made with an API token: what it may do is decided per kind. */
|
|
28
|
-
token?: { scopes: readonly string[] } | undefined
|
|
29
|
-
/** Also delete the objects of the kinds the documents hold that the documents do not mention. Off by default. */
|
|
30
|
-
prune?: boolean
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
/** What a kind sees while it reads the documents of a file. */
|
|
34
|
-
export interface PlanContext<State> {
|
|
35
|
-
state: State
|
|
36
|
-
token: { scopes: readonly string[] } | undefined
|
|
37
|
-
problems: Problems
|
|
38
|
-
/** `{ vault: <name>, secret: <name> }` references in a config, turned into the ids of this installation. */
|
|
39
|
-
resolveVaults: (value: unknown, path: string) => unknown
|
|
40
|
-
/** The documents of a kind the file holds, as that kind parsed them. */
|
|
41
|
-
desired: (kind: string) => unknown[]
|
|
42
|
-
/** The keys of a kind that an import with `prune` deletes: they do not exist afterwards, so nothing may link to them. */
|
|
43
|
-
removed: (kind: string) => ReadonlySet<string>
|
|
44
|
-
}
|
|
45
|
-
|
|
46
|
-
/** What a kind sees while it writes. */
|
|
47
|
-
export interface ApplyContext<State> {
|
|
48
|
-
state: State
|
|
49
|
-
now: number
|
|
50
|
-
/** Ids by key, per kind: the objects that exist now, plus those created so far in this run. */
|
|
51
|
-
ids: (kind: string) => Map<string, number>
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
/**
|
|
55
|
-
* One kind of document. `parse` reads a document of the kind (reporting problems), `plan` compares what the file
|
|
56
|
-
* holds with the installation and says what would change, `apply` writes it, `link` runs after every kind is written
|
|
57
|
-
* (for references to objects that might be created later in the same file), `export` writes the kind out.
|
|
58
|
-
*/
|
|
59
|
-
export interface ConfigKind<State, ExportState, Desired = any, Planned = any> { // eslint-disable-line @typescript-eslint/no-explicit-any
|
|
60
|
-
kind: string
|
|
61
|
-
parse: (entry: Json, index: number, seen: Set<string>, problems: Problems) => Desired | null
|
|
62
|
-
plan: (desired: Desired[], ctx: PlanContext<State>) => { planned: Planned; changes: ConfigChange[] }
|
|
63
|
-
/** The ids of the objects of this kind that exist, by key. */
|
|
64
|
-
existingIds: (state: State) => Map<string, number>
|
|
65
|
-
apply: (planned: Planned, ctx: ApplyContext<State>) => Promise<AuditEntry[]>
|
|
66
|
-
link?: (planned: Planned, ctx: ApplyContext<State>) => Promise<void>
|
|
67
|
-
/** Deletes the objects with these keys, and whatever links to them. A kind without it is never pruned. */
|
|
68
|
-
remove?: (keys: string[], ctx: ApplyContext<State>) => Promise<AuditEntry[]>
|
|
69
|
-
/** The documents of this kind, sorted by key. `only` limits them to one key (and nothing else is written out). */
|
|
70
|
-
export: (state: ExportState, ctx: ExportContext) => Json[]
|
|
71
|
-
}
|
|
72
|
-
|
|
73
|
-
export interface ExportContext {
|
|
74
|
-
/** Replaces the vault references of a config with the names of the vault and secret. */
|
|
75
|
-
exportVaultRefs: (value: unknown) => unknown
|
|
76
|
-
/** The rows to write out: all of them, or only the one asked for. */
|
|
77
|
-
wanted: <T extends { key: string }>(rows: T[], kind: string) => T[]
|
|
78
|
-
/** Whether the kind is part of this export (a kind may be left out for a reader who may not see it). */
|
|
79
|
-
included: (kind: string) => boolean
|
|
80
|
-
}
|
|
81
|
-
|
|
82
|
-
export interface ConfigEngineOptions<State, ExportState, Actor> {
|
|
83
|
-
/** The kinds, in the order messages name them. */
|
|
84
|
-
kinds: ReadonlyArray<ConfigKind<State, ExportState>>
|
|
85
|
-
/** The order kinds are planned, written and exported in (kind names); defaults to `kinds`. */
|
|
86
|
-
order?: readonly string[]
|
|
87
|
-
/** Reads what the installation holds. Called inside the transaction when the file is applied. */
|
|
88
|
-
loadState: () => Promise<State>
|
|
89
|
-
/** Reads what an export needs; a kind not in `included` need not be read. */
|
|
90
|
-
loadExportState: (included: ReadonlySet<string>) => Promise<ExportState>
|
|
91
|
-
vaultRows: (state: State) => VaultRows
|
|
92
|
-
exportVaultRows: (state: ExportState) => VaultRows
|
|
93
|
-
/** Runs `work` so that everything it writes happens, or none of it does. */
|
|
94
|
-
transaction: <T>(work: () => Promise<T>) => Promise<T>
|
|
95
|
-
audit: { write: (actor: Actor, action: 'create' | 'update' | 'delete', entity: string, id: number | string | null, name: string, diff?: Json) => Promise<void> }
|
|
96
|
-
/** Runs once after a successful apply that changed something, before the audit entries are written. */
|
|
97
|
-
afterApply?: () => Promise<void>
|
|
98
|
-
/** Entity type of the summary entry written to the audit log. */
|
|
99
|
-
auditEntity?: string
|
|
100
|
-
}
|
|
101
|
-
|
|
102
|
-
/** Makes the installation match the documents, completely or not at all, and writes it out again as documents. */
|
|
103
|
-
export function createConfigEngine<State, ExportState, Actor>(options: ConfigEngineOptions<State, ExportState, Actor>) {
|
|
104
|
-
const kinds = new Map(options.kinds.map((kind) => [kind.kind, kind]))
|
|
105
|
-
const names = options.kinds.map((kind) => kind.kind)
|
|
106
|
-
const order = (options.order ?? names).map((name) => kinds.get(name)!)
|
|
107
|
-
|
|
108
|
-
/** The kinds a list of documents contains, so that the caller can check it may import each. Documents of no known kind are ignored here. */
|
|
109
|
-
function kindsIn(documents: unknown[]): Set<string> {
|
|
110
|
-
const found = new Set<string>()
|
|
111
|
-
for (const document of documents) {
|
|
112
|
-
if (isObject(document) && names.includes(document['kind'] as string)) found.add(document['kind'] as string)
|
|
113
|
-
}
|
|
114
|
-
return found
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
function parseDocuments(documents: unknown[], problems: Problems): Map<string, unknown[]> {
|
|
118
|
-
const desired = new Map<string, unknown[]>(names.map((name) => [name, []]))
|
|
119
|
-
const seenKeys = new Map<string, Set<string>>(names.map((name) => [name, new Set()]))
|
|
120
|
-
|
|
121
|
-
documents.forEach((document, index) => {
|
|
122
|
-
const label = `document ${index + 1}`
|
|
123
|
-
const poisoned = forbiddenNameAt(document, label)
|
|
124
|
-
if (poisoned) { problems.add(poisoned, '"__proto__" is not allowed as a name'); return }
|
|
125
|
-
if (!isObject(document)) { problems.add(label, 'must be an object with a kind'); return }
|
|
126
|
-
|
|
127
|
-
const kind = kinds.get(document['kind'] as string)
|
|
128
|
-
if (!kind) {
|
|
129
|
-
problems.add(`${label}.kind`, document['kind'] === undefined
|
|
130
|
-
? `is required: one of ${names.join(', ')}`
|
|
131
|
-
: `"${clip(String(document['kind']))}" is not a kind; use one of ${names.join(', ')}`)
|
|
132
|
-
return
|
|
133
|
-
}
|
|
134
|
-
const parsed = kind.parse(document, index, seenKeys.get(kind.kind)!, problems)
|
|
135
|
-
if (parsed) desired.get(kind.kind)!.push(parsed)
|
|
136
|
-
})
|
|
137
|
-
return desired
|
|
138
|
-
}
|
|
139
|
-
|
|
140
|
-
/** The keys to delete, per kind. A kind with no document in the file is left alone, so an empty file prunes nothing. */
|
|
141
|
-
function stale(state: State, desired: Map<string, unknown[]>): Map<string, string[]> {
|
|
142
|
-
const found = new Map<string, string[]>()
|
|
143
|
-
for (const kind of order) {
|
|
144
|
-
const documents = desired.get(kind.kind) ?? []
|
|
145
|
-
if (!kind.remove || documents.length === 0) continue
|
|
146
|
-
const wanted = new Set(documents.map((document) => (document as { key: string }).key))
|
|
147
|
-
const keys = [...kind.existingIds(state).keys()].filter((key) => !wanted.has(key)).sort()
|
|
148
|
-
if (keys.length) found.set(kind.kind, keys)
|
|
149
|
-
}
|
|
150
|
-
return found
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
function plan(state: State, desired: Map<string, unknown[]>, token: ImportOptions<Actor>['token'], prune: boolean) {
|
|
154
|
-
const problems = new Problems()
|
|
155
|
-
const resolve = createVaultResolver(options.vaultRows(state))
|
|
156
|
-
const removals = prune ? stale(state, desired) : new Map<string, string[]>()
|
|
157
|
-
const ctx: PlanContext<State> = {
|
|
158
|
-
state, token, problems,
|
|
159
|
-
resolveVaults: (value, path) => resolve(value, path, problems),
|
|
160
|
-
desired: (kind) => desired.get(kind) ?? [],
|
|
161
|
-
removed: (kind) => new Set(removals.get(kind) ?? []),
|
|
162
|
-
}
|
|
163
|
-
const planned = new Map<string, unknown>()
|
|
164
|
-
const changes: ConfigChange[] = []
|
|
165
|
-
for (const kind of order) {
|
|
166
|
-
const result = kind.plan(desired.get(kind.kind) ?? [], ctx)
|
|
167
|
-
planned.set(kind.kind, result.planned)
|
|
168
|
-
changes.push(...result.changes)
|
|
169
|
-
}
|
|
170
|
-
if (problems.any) throw new ConfigInvalidError(problems.list)
|
|
171
|
-
for (const kind of order) for (const key of removals.get(kind.kind) ?? []) changes.push({ kind: kind.kind, key, action: 'delete' })
|
|
172
|
-
return { planned, changes, removals }
|
|
173
|
-
}
|
|
174
|
-
|
|
175
|
-
async function execute(state: State, planned: Map<string, unknown>, removals: Map<string, string[]>): Promise<AuditEntry[]> {
|
|
176
|
-
const ids = new Map(names.map((name) => [name, kinds.get(name)!.existingIds(state)]))
|
|
177
|
-
const ctx: ApplyContext<State> = { state, now: Date.now(), ids: (kind) => ids.get(kind)! }
|
|
178
|
-
const audit: AuditEntry[] = []
|
|
179
|
-
for (const kind of order) audit.push(...await kind.apply(planned.get(kind.kind), ctx))
|
|
180
|
-
for (const kind of order) await kind.link?.(planned.get(kind.kind), ctx)
|
|
181
|
-
// Last, and in the reverse order, so that what links to a kind is deleted before the kind.
|
|
182
|
-
for (const kind of [...order].reverse()) {
|
|
183
|
-
const keys = removals.get(kind.kind)
|
|
184
|
-
if (keys) audit.push(...await kind.remove!(keys, ctx))
|
|
185
|
-
}
|
|
186
|
-
return audit
|
|
187
|
-
}
|
|
188
|
-
|
|
189
|
-
function summarise(changes: ConfigChange[], prune: boolean): ImportResult['summary'] {
|
|
190
|
-
const summary: Record<ChangeAction, number> = { create: 0, update: 0, delete: 0, unchanged: 0 }
|
|
191
|
-
for (const change of changes) summary[change.action] += 1
|
|
192
|
-
if (!prune) return { create: summary.create, update: summary.update, unchanged: summary.unchanged }
|
|
193
|
-
return summary
|
|
194
|
-
}
|
|
195
|
-
|
|
196
|
-
/**
|
|
197
|
-
* Makes the installation match the documents: an object whose key exists is updated, one whose key does not is created.
|
|
198
|
-
* Nothing is removed unless `prune` is set. Everything is checked before anything is written, and the writes are one transaction, so a
|
|
199
|
-
* stream is applied completely or not at all. With `dryRun` nothing is written and the result says what would happen.
|
|
200
|
-
*/
|
|
201
|
-
async function importDocuments(documents: unknown[], importOptions: ImportOptions<Actor>): Promise<ImportResult> {
|
|
202
|
-
const problems = new Problems()
|
|
203
|
-
const desired = parseDocuments(documents, problems)
|
|
204
|
-
if (problems.any) throw new ConfigInvalidError(problems.list)
|
|
205
|
-
|
|
206
|
-
const result = (changes: ConfigChange[]): ImportResult => ({ dryRun: importOptions.dryRun, summary: summarise(changes, !!importOptions.prune), changes })
|
|
207
|
-
|
|
208
|
-
if (importOptions.dryRun) return result(plan(await options.loadState(), desired, importOptions.token, !!importOptions.prune).changes)
|
|
209
|
-
|
|
210
|
-
const { changes, audit } = await options.transaction(async () => {
|
|
211
|
-
const state = await options.loadState()
|
|
212
|
-
const { planned, changes, removals } = plan(state, desired, importOptions.token, !!importOptions.prune)
|
|
213
|
-
const audit = changes.some((change) => change.action !== 'unchanged') ? await execute(state, planned, removals) : []
|
|
214
|
-
return { changes, audit }
|
|
215
|
-
})
|
|
216
|
-
|
|
217
|
-
// The changes are committed. A failure past this point must not turn a successful import into an error (the
|
|
218
|
-
// client would retry, and a second apply is a no-op), so it is logged instead.
|
|
219
|
-
if (audit.length) {
|
|
220
|
-
try {
|
|
221
|
-
await options.afterApply?.()
|
|
222
|
-
for (const entry of audit) await options.audit.write(importOptions.actor, entry.action, entry.entity, entry.id, entry.name, entry.diff)
|
|
223
|
-
const summary = summarise(changes, !!importOptions.prune)
|
|
224
|
-
await options.audit.write(importOptions.actor, 'update', options.auditEntity ?? 'config', null, 'Configuration import', {
|
|
225
|
-
created: summary.create, updated: summary.update, ...(importOptions.prune ? { deleted: summary.delete ?? 0 } : {}), unchanged: summary.unchanged,
|
|
226
|
-
})
|
|
227
|
-
} catch (error) {
|
|
228
|
-
console.error('[config] Imported, but could not finish the follow-up work:', error)
|
|
229
|
-
}
|
|
230
|
-
}
|
|
231
|
-
return result(changes)
|
|
232
|
-
}
|
|
233
|
-
|
|
234
|
-
/**
|
|
235
|
-
* The documents of every kind, in a fixed order that does not depend on this installation's numeric ids. Secrets are
|
|
236
|
-
* masked by the kinds, and runtime state is left out. `only` writes out one document; `included` leaves out kinds.
|
|
237
|
-
*/
|
|
238
|
-
async function exportDocuments(only?: { kind: string; key: string }, exportOptions: { included?: ReadonlySet<string> } = {}): Promise<Json[]> {
|
|
239
|
-
const included = exportOptions.included ?? new Set(names)
|
|
240
|
-
const state = await options.loadExportState(included)
|
|
241
|
-
const ctx: ExportContext = {
|
|
242
|
-
exportVaultRefs: createVaultExporter(options.exportVaultRows(state)),
|
|
243
|
-
// Only the rows asked for are written out, so a vault name that is ambiguous for some other object does not matter.
|
|
244
|
-
wanted: (rows, kind) => (only ? rows.filter((row) => only.kind === kind && row.key === only.key) : rows),
|
|
245
|
-
included: (kind) => included.has(kind),
|
|
246
|
-
}
|
|
247
|
-
return order.filter((kind) => included.has(kind.kind)).flatMap((kind) => kind.export(state, ctx))
|
|
248
|
-
}
|
|
249
|
-
|
|
250
|
-
/** The one document of a kind and key, or undefined when there is none. */
|
|
251
|
-
function pickDocument(documents: Json[], kind: string, key: string | undefined): Json | undefined {
|
|
252
|
-
return documents.find((document) => document['kind'] === kind && document['key'] === key)
|
|
253
|
-
}
|
|
254
|
-
|
|
255
|
-
return { kinds: names, kindsIn, importDocuments, exportDocuments, pickDocument }
|
|
256
|
-
}
|
|
257
|
-
|
|
258
|
-
export type ConfigEngine<State, ExportState, Actor> = ReturnType<typeof createConfigEngine<State, ExportState, Actor>>
|
|
1
|
+
import { clip, ConfigInvalidError, createVaultExporter, createVaultResolver, forbiddenNameAt, isObject, Problems, type Json, type VaultRows } from './document.js'
|
|
2
|
+
|
|
3
|
+
export type ChangeAction = 'create' | 'update' | 'delete' | 'unchanged'
|
|
4
|
+
|
|
5
|
+
export interface ConfigChange {
|
|
6
|
+
kind: string
|
|
7
|
+
/** The key of the object the document describes. */
|
|
8
|
+
key?: string
|
|
9
|
+
action: ChangeAction
|
|
10
|
+
/** For an update: which settings differ. Never their values, which may be secrets. */
|
|
11
|
+
fields?: string[]
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export interface ImportResult {
|
|
15
|
+
dryRun: boolean
|
|
16
|
+
/** `delete` is present only for an import with `prune`. */
|
|
17
|
+
summary: Record<'create' | 'update' | 'unchanged', number> & { delete?: number }
|
|
18
|
+
changes: ConfigChange[]
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** What one change did, for the audit log. */
|
|
22
|
+
export interface AuditEntry { action: 'create' | 'update' | 'delete'; entity: string; id: number | string; name: string; diff: Json }
|
|
23
|
+
|
|
24
|
+
export interface ImportOptions<Actor> {
|
|
25
|
+
dryRun: boolean
|
|
26
|
+
actor: Actor
|
|
27
|
+
/** Set when the request is made with an API token: what it may do is decided per kind. */
|
|
28
|
+
token?: { scopes: readonly string[] } | undefined
|
|
29
|
+
/** Also delete the objects of the kinds the documents hold that the documents do not mention. Off by default. */
|
|
30
|
+
prune?: boolean
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** What a kind sees while it reads the documents of a file. */
|
|
34
|
+
export interface PlanContext<State> {
|
|
35
|
+
state: State
|
|
36
|
+
token: { scopes: readonly string[] } | undefined
|
|
37
|
+
problems: Problems
|
|
38
|
+
/** `{ vault: <name>, secret: <name> }` references in a config, turned into the ids of this installation. */
|
|
39
|
+
resolveVaults: (value: unknown, path: string) => unknown
|
|
40
|
+
/** The documents of a kind the file holds, as that kind parsed them. */
|
|
41
|
+
desired: (kind: string) => unknown[]
|
|
42
|
+
/** The keys of a kind that an import with `prune` deletes: they do not exist afterwards, so nothing may link to them. */
|
|
43
|
+
removed: (kind: string) => ReadonlySet<string>
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/** What a kind sees while it writes. */
|
|
47
|
+
export interface ApplyContext<State> {
|
|
48
|
+
state: State
|
|
49
|
+
now: number
|
|
50
|
+
/** Ids by key, per kind: the objects that exist now, plus those created so far in this run. */
|
|
51
|
+
ids: (kind: string) => Map<string, number>
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* One kind of document. `parse` reads a document of the kind (reporting problems), `plan` compares what the file
|
|
56
|
+
* holds with the installation and says what would change, `apply` writes it, `link` runs after every kind is written
|
|
57
|
+
* (for references to objects that might be created later in the same file), `export` writes the kind out.
|
|
58
|
+
*/
|
|
59
|
+
export interface ConfigKind<State, ExportState, Desired = any, Planned = any> { // eslint-disable-line @typescript-eslint/no-explicit-any
|
|
60
|
+
kind: string
|
|
61
|
+
parse: (entry: Json, index: number, seen: Set<string>, problems: Problems) => Desired | null
|
|
62
|
+
plan: (desired: Desired[], ctx: PlanContext<State>) => { planned: Planned; changes: ConfigChange[] }
|
|
63
|
+
/** The ids of the objects of this kind that exist, by key. */
|
|
64
|
+
existingIds: (state: State) => Map<string, number>
|
|
65
|
+
apply: (planned: Planned, ctx: ApplyContext<State>) => Promise<AuditEntry[]>
|
|
66
|
+
link?: (planned: Planned, ctx: ApplyContext<State>) => Promise<void>
|
|
67
|
+
/** Deletes the objects with these keys, and whatever links to them. A kind without it is never pruned. */
|
|
68
|
+
remove?: (keys: string[], ctx: ApplyContext<State>) => Promise<AuditEntry[]>
|
|
69
|
+
/** The documents of this kind, sorted by key. `only` limits them to one key (and nothing else is written out). */
|
|
70
|
+
export: (state: ExportState, ctx: ExportContext) => Json[]
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
export interface ExportContext {
|
|
74
|
+
/** Replaces the vault references of a config with the names of the vault and secret. */
|
|
75
|
+
exportVaultRefs: (value: unknown) => unknown
|
|
76
|
+
/** The rows to write out: all of them, or only the one asked for. */
|
|
77
|
+
wanted: <T extends { key: string }>(rows: T[], kind: string) => T[]
|
|
78
|
+
/** Whether the kind is part of this export (a kind may be left out for a reader who may not see it). */
|
|
79
|
+
included: (kind: string) => boolean
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
export interface ConfigEngineOptions<State, ExportState, Actor> {
|
|
83
|
+
/** The kinds, in the order messages name them. */
|
|
84
|
+
kinds: ReadonlyArray<ConfigKind<State, ExportState>>
|
|
85
|
+
/** The order kinds are planned, written and exported in (kind names); defaults to `kinds`. */
|
|
86
|
+
order?: readonly string[]
|
|
87
|
+
/** Reads what the installation holds. Called inside the transaction when the file is applied. */
|
|
88
|
+
loadState: () => Promise<State>
|
|
89
|
+
/** Reads what an export needs; a kind not in `included` need not be read. */
|
|
90
|
+
loadExportState: (included: ReadonlySet<string>) => Promise<ExportState>
|
|
91
|
+
vaultRows: (state: State) => VaultRows
|
|
92
|
+
exportVaultRows: (state: ExportState) => VaultRows
|
|
93
|
+
/** Runs `work` so that everything it writes happens, or none of it does. */
|
|
94
|
+
transaction: <T>(work: () => Promise<T>) => Promise<T>
|
|
95
|
+
audit: { write: (actor: Actor, action: 'create' | 'update' | 'delete', entity: string, id: number | string | null, name: string, diff?: Json) => Promise<void> }
|
|
96
|
+
/** Runs once after a successful apply that changed something, before the audit entries are written. */
|
|
97
|
+
afterApply?: () => Promise<void>
|
|
98
|
+
/** Entity type of the summary entry written to the audit log. */
|
|
99
|
+
auditEntity?: string
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
/** Makes the installation match the documents, completely or not at all, and writes it out again as documents. */
|
|
103
|
+
export function createConfigEngine<State, ExportState, Actor>(options: ConfigEngineOptions<State, ExportState, Actor>) {
|
|
104
|
+
const kinds = new Map(options.kinds.map((kind) => [kind.kind, kind]))
|
|
105
|
+
const names = options.kinds.map((kind) => kind.kind)
|
|
106
|
+
const order = (options.order ?? names).map((name) => kinds.get(name)!)
|
|
107
|
+
|
|
108
|
+
/** The kinds a list of documents contains, so that the caller can check it may import each. Documents of no known kind are ignored here. */
|
|
109
|
+
function kindsIn(documents: unknown[]): Set<string> {
|
|
110
|
+
const found = new Set<string>()
|
|
111
|
+
for (const document of documents) {
|
|
112
|
+
if (isObject(document) && names.includes(document['kind'] as string)) found.add(document['kind'] as string)
|
|
113
|
+
}
|
|
114
|
+
return found
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function parseDocuments(documents: unknown[], problems: Problems): Map<string, unknown[]> {
|
|
118
|
+
const desired = new Map<string, unknown[]>(names.map((name) => [name, []]))
|
|
119
|
+
const seenKeys = new Map<string, Set<string>>(names.map((name) => [name, new Set()]))
|
|
120
|
+
|
|
121
|
+
documents.forEach((document, index) => {
|
|
122
|
+
const label = `document ${index + 1}`
|
|
123
|
+
const poisoned = forbiddenNameAt(document, label)
|
|
124
|
+
if (poisoned) { problems.add(poisoned, '"__proto__" is not allowed as a name'); return }
|
|
125
|
+
if (!isObject(document)) { problems.add(label, 'must be an object with a kind'); return }
|
|
126
|
+
|
|
127
|
+
const kind = kinds.get(document['kind'] as string)
|
|
128
|
+
if (!kind) {
|
|
129
|
+
problems.add(`${label}.kind`, document['kind'] === undefined
|
|
130
|
+
? `is required: one of ${names.join(', ')}`
|
|
131
|
+
: `"${clip(String(document['kind']))}" is not a kind; use one of ${names.join(', ')}`)
|
|
132
|
+
return
|
|
133
|
+
}
|
|
134
|
+
const parsed = kind.parse(document, index, seenKeys.get(kind.kind)!, problems)
|
|
135
|
+
if (parsed) desired.get(kind.kind)!.push(parsed)
|
|
136
|
+
})
|
|
137
|
+
return desired
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/** The keys to delete, per kind. A kind with no document in the file is left alone, so an empty file prunes nothing. */
|
|
141
|
+
function stale(state: State, desired: Map<string, unknown[]>): Map<string, string[]> {
|
|
142
|
+
const found = new Map<string, string[]>()
|
|
143
|
+
for (const kind of order) {
|
|
144
|
+
const documents = desired.get(kind.kind) ?? []
|
|
145
|
+
if (!kind.remove || documents.length === 0) continue
|
|
146
|
+
const wanted = new Set(documents.map((document) => (document as { key: string }).key))
|
|
147
|
+
const keys = [...kind.existingIds(state).keys()].filter((key) => !wanted.has(key)).sort()
|
|
148
|
+
if (keys.length) found.set(kind.kind, keys)
|
|
149
|
+
}
|
|
150
|
+
return found
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
function plan(state: State, desired: Map<string, unknown[]>, token: ImportOptions<Actor>['token'], prune: boolean) {
|
|
154
|
+
const problems = new Problems()
|
|
155
|
+
const resolve = createVaultResolver(options.vaultRows(state))
|
|
156
|
+
const removals = prune ? stale(state, desired) : new Map<string, string[]>()
|
|
157
|
+
const ctx: PlanContext<State> = {
|
|
158
|
+
state, token, problems,
|
|
159
|
+
resolveVaults: (value, path) => resolve(value, path, problems),
|
|
160
|
+
desired: (kind) => desired.get(kind) ?? [],
|
|
161
|
+
removed: (kind) => new Set(removals.get(kind) ?? []),
|
|
162
|
+
}
|
|
163
|
+
const planned = new Map<string, unknown>()
|
|
164
|
+
const changes: ConfigChange[] = []
|
|
165
|
+
for (const kind of order) {
|
|
166
|
+
const result = kind.plan(desired.get(kind.kind) ?? [], ctx)
|
|
167
|
+
planned.set(kind.kind, result.planned)
|
|
168
|
+
changes.push(...result.changes)
|
|
169
|
+
}
|
|
170
|
+
if (problems.any) throw new ConfigInvalidError(problems.list)
|
|
171
|
+
for (const kind of order) for (const key of removals.get(kind.kind) ?? []) changes.push({ kind: kind.kind, key, action: 'delete' })
|
|
172
|
+
return { planned, changes, removals }
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
async function execute(state: State, planned: Map<string, unknown>, removals: Map<string, string[]>): Promise<AuditEntry[]> {
|
|
176
|
+
const ids = new Map(names.map((name) => [name, kinds.get(name)!.existingIds(state)]))
|
|
177
|
+
const ctx: ApplyContext<State> = { state, now: Date.now(), ids: (kind) => ids.get(kind)! }
|
|
178
|
+
const audit: AuditEntry[] = []
|
|
179
|
+
for (const kind of order) audit.push(...await kind.apply(planned.get(kind.kind), ctx))
|
|
180
|
+
for (const kind of order) await kind.link?.(planned.get(kind.kind), ctx)
|
|
181
|
+
// Last, and in the reverse order, so that what links to a kind is deleted before the kind.
|
|
182
|
+
for (const kind of [...order].reverse()) {
|
|
183
|
+
const keys = removals.get(kind.kind)
|
|
184
|
+
if (keys) audit.push(...await kind.remove!(keys, ctx))
|
|
185
|
+
}
|
|
186
|
+
return audit
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
function summarise(changes: ConfigChange[], prune: boolean): ImportResult['summary'] {
|
|
190
|
+
const summary: Record<ChangeAction, number> = { create: 0, update: 0, delete: 0, unchanged: 0 }
|
|
191
|
+
for (const change of changes) summary[change.action] += 1
|
|
192
|
+
if (!prune) return { create: summary.create, update: summary.update, unchanged: summary.unchanged }
|
|
193
|
+
return summary
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
/**
|
|
197
|
+
* Makes the installation match the documents: an object whose key exists is updated, one whose key does not is created.
|
|
198
|
+
* Nothing is removed unless `prune` is set. Everything is checked before anything is written, and the writes are one transaction, so a
|
|
199
|
+
* stream is applied completely or not at all. With `dryRun` nothing is written and the result says what would happen.
|
|
200
|
+
*/
|
|
201
|
+
async function importDocuments(documents: unknown[], importOptions: ImportOptions<Actor>): Promise<ImportResult> {
|
|
202
|
+
const problems = new Problems()
|
|
203
|
+
const desired = parseDocuments(documents, problems)
|
|
204
|
+
if (problems.any) throw new ConfigInvalidError(problems.list)
|
|
205
|
+
|
|
206
|
+
const result = (changes: ConfigChange[]): ImportResult => ({ dryRun: importOptions.dryRun, summary: summarise(changes, !!importOptions.prune), changes })
|
|
207
|
+
|
|
208
|
+
if (importOptions.dryRun) return result(plan(await options.loadState(), desired, importOptions.token, !!importOptions.prune).changes)
|
|
209
|
+
|
|
210
|
+
const { changes, audit } = await options.transaction(async () => {
|
|
211
|
+
const state = await options.loadState()
|
|
212
|
+
const { planned, changes, removals } = plan(state, desired, importOptions.token, !!importOptions.prune)
|
|
213
|
+
const audit = changes.some((change) => change.action !== 'unchanged') ? await execute(state, planned, removals) : []
|
|
214
|
+
return { changes, audit }
|
|
215
|
+
})
|
|
216
|
+
|
|
217
|
+
// The changes are committed. A failure past this point must not turn a successful import into an error (the
|
|
218
|
+
// client would retry, and a second apply is a no-op), so it is logged instead.
|
|
219
|
+
if (audit.length) {
|
|
220
|
+
try {
|
|
221
|
+
await options.afterApply?.()
|
|
222
|
+
for (const entry of audit) await options.audit.write(importOptions.actor, entry.action, entry.entity, entry.id, entry.name, entry.diff)
|
|
223
|
+
const summary = summarise(changes, !!importOptions.prune)
|
|
224
|
+
await options.audit.write(importOptions.actor, 'update', options.auditEntity ?? 'config', null, 'Configuration import', {
|
|
225
|
+
created: summary.create, updated: summary.update, ...(importOptions.prune ? { deleted: summary.delete ?? 0 } : {}), unchanged: summary.unchanged,
|
|
226
|
+
})
|
|
227
|
+
} catch (error) {
|
|
228
|
+
console.error('[config] Imported, but could not finish the follow-up work:', error)
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
return result(changes)
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
/**
|
|
235
|
+
* The documents of every kind, in a fixed order that does not depend on this installation's numeric ids. Secrets are
|
|
236
|
+
* masked by the kinds, and runtime state is left out. `only` writes out one document; `included` leaves out kinds.
|
|
237
|
+
*/
|
|
238
|
+
async function exportDocuments(only?: { kind: string; key: string }, exportOptions: { included?: ReadonlySet<string> } = {}): Promise<Json[]> {
|
|
239
|
+
const included = exportOptions.included ?? new Set(names)
|
|
240
|
+
const state = await options.loadExportState(included)
|
|
241
|
+
const ctx: ExportContext = {
|
|
242
|
+
exportVaultRefs: createVaultExporter(options.exportVaultRows(state)),
|
|
243
|
+
// Only the rows asked for are written out, so a vault name that is ambiguous for some other object does not matter.
|
|
244
|
+
wanted: (rows, kind) => (only ? rows.filter((row) => only.kind === kind && row.key === only.key) : rows),
|
|
245
|
+
included: (kind) => included.has(kind),
|
|
246
|
+
}
|
|
247
|
+
return order.filter((kind) => included.has(kind.kind)).flatMap((kind) => kind.export(state, ctx))
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/** The one document of a kind and key, or undefined when there is none. */
|
|
251
|
+
function pickDocument(documents: Json[], kind: string, key: string | undefined): Json | undefined {
|
|
252
|
+
return documents.find((document) => document['kind'] === kind && document['key'] === key)
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
return { kinds: names, kindsIn, importDocuments, exportDocuments, pickDocument }
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
export type ConfigEngine<State, ExportState, Actor> = ReturnType<typeof createConfigEngine<State, ExportState, Actor>>
|
package/src/index.ts
CHANGED
|
@@ -1,16 +1,16 @@
|
|
|
1
|
-
export {
|
|
2
|
-
backfillKeys, ENTITY_KEY_ERROR, ENTITY_KEY_PATTERN, isValidEntityKey, keyForNew, randomEntityKey, slugifyKey,
|
|
3
|
-
type KeyOwner, type KeyResult,
|
|
4
|
-
} from './keys.js'
|
|
5
|
-
export { MAX_DOCUMENTS, readDocuments, toYaml, toYamlStream } from './yaml.js'
|
|
6
|
-
export {
|
|
7
|
-
byKey, canonical, clip, ConfigExportError, ConfigInvalidError, createVaultExporter, createVaultResolver, documentKey, findCycle, forbiddenNameAt,
|
|
8
|
-
isObject, keyList, nestedDeeperThan, Problems, unknownFields,
|
|
9
|
-
type ConfigProblem, type ConfigVaultRef, type Json, type VaultRows,
|
|
10
|
-
} from './document.js'
|
|
11
|
-
export {
|
|
12
|
-
createConfigEngine,
|
|
13
|
-
type ApplyContext, type AuditEntry, type ChangeAction, type ConfigChange, type ConfigEngine, type ConfigEngineOptions, type ConfigKind,
|
|
14
|
-
type ExportContext, type ImportOptions, type ImportResult, type PlanContext,
|
|
15
|
-
} from './engine.js'
|
|
16
|
-
export { createConfigRoutes, type ConfigCaller, type ConfigRoutesOptions, type KindAccess } from './routes.js'
|
|
1
|
+
export {
|
|
2
|
+
backfillKeys, ENTITY_KEY_ERROR, ENTITY_KEY_PATTERN, isValidEntityKey, keyForNew, randomEntityKey, slugifyKey,
|
|
3
|
+
type KeyOwner, type KeyResult,
|
|
4
|
+
} from './keys.js'
|
|
5
|
+
export { MAX_DOCUMENTS, readDocuments, toYaml, toYamlStream } from './yaml.js'
|
|
6
|
+
export {
|
|
7
|
+
byKey, canonical, clip, ConfigExportError, ConfigInvalidError, createVaultExporter, createVaultResolver, documentKey, findCycle, forbiddenNameAt,
|
|
8
|
+
isObject, keyList, nestedDeeperThan, Problems, unknownFields,
|
|
9
|
+
type ConfigProblem, type ConfigVaultRef, type Json, type VaultRows,
|
|
10
|
+
} from './document.js'
|
|
11
|
+
export {
|
|
12
|
+
createConfigEngine,
|
|
13
|
+
type ApplyContext, type AuditEntry, type ChangeAction, type ConfigChange, type ConfigEngine, type ConfigEngineOptions, type ConfigKind,
|
|
14
|
+
type ExportContext, type ImportOptions, type ImportResult, type PlanContext,
|
|
15
|
+
} from './engine.js'
|
|
16
|
+
export { createConfigRoutes, type ConfigCaller, type ConfigRoutesOptions, type KindAccess } from './routes.js'
|