@meith/cli 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +165 -0
- package/bin/community.mjs +117 -0
- package/package.json +47 -0
- package/src/args.ts +63 -0
- package/src/backup.ts +677 -0
- package/src/board-eject.ts +180 -0
- package/src/commands.ts +269 -0
- package/src/context.ts +77 -0
- package/src/demo.ts +61 -0
- package/src/import-files.ts +189 -0
- package/src/import.ts +142 -0
- package/src/index.ts +394 -0
- package/src/plugin-manifest.ts +176 -0
- package/src/plugins.ts +65 -0
- package/src/profile-fields.ts +107 -0
- package/src/push.ts +47 -0
- package/src/redaction.ts +23 -0
- package/src/search.ts +33 -0
- package/src/tasks.ts +69 -0
- package/src/upgrade.ts +125 -0
package/src/backup.ts
ADDED
|
@@ -0,0 +1,677 @@
|
|
|
1
|
+
import { spawn } from 'node:child_process'
|
|
2
|
+
import {
|
|
3
|
+
chmod,
|
|
4
|
+
copyFile,
|
|
5
|
+
mkdir,
|
|
6
|
+
mkdtemp,
|
|
7
|
+
open,
|
|
8
|
+
readdir,
|
|
9
|
+
readFile,
|
|
10
|
+
rm,
|
|
11
|
+
stat,
|
|
12
|
+
writeFile,
|
|
13
|
+
} from 'node:fs/promises'
|
|
14
|
+
import { tmpdir } from 'node:os'
|
|
15
|
+
import path from 'node:path'
|
|
16
|
+
|
|
17
|
+
import { ConfigurationError, env, ValidationError } from '@meith/core'
|
|
18
|
+
import { migrationUrl, runMigrations } from '@meith/db'
|
|
19
|
+
import { S3FileStore } from '@meith/drivers'
|
|
20
|
+
|
|
21
|
+
import { optional, parseFlags } from './args'
|
|
22
|
+
import { requirePostgres } from './context'
|
|
23
|
+
import { CODE_VERSION } from './upgrade'
|
|
24
|
+
|
|
25
|
+
export type UploadsMode = 'include' | 'skip'
|
|
26
|
+
|
|
27
|
+
export interface BackupManifest {
|
|
28
|
+
readonly format: 1
|
|
29
|
+
readonly createdAt: string
|
|
30
|
+
readonly version: string
|
|
31
|
+
readonly filestore: 'local' | 's3'
|
|
32
|
+
readonly uploads: 'included' | 'skipped'
|
|
33
|
+
readonly bucket?: string
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function resolveUploadsMode(driver: 'local' | 's3', flag: string | undefined): UploadsMode {
|
|
37
|
+
if (flag === undefined) return driver === 'local' ? 'include' : 'skip'
|
|
38
|
+
if (flag === 'include' || flag === 'skip') return flag
|
|
39
|
+
throw new ValidationError(`--uploads must be "include" or "skip", got "${flag}".`)
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export function bundleName(at: Date): string {
|
|
43
|
+
const stamp = at
|
|
44
|
+
.toISOString()
|
|
45
|
+
.replace(/\.\d+Z$/, 'Z')
|
|
46
|
+
.replaceAll(':', '-')
|
|
47
|
+
return `meith-backup-${stamp}.tar.gz`
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export function parseManifest(raw: string): BackupManifest {
|
|
51
|
+
let parsed: unknown
|
|
52
|
+
try {
|
|
53
|
+
parsed = JSON.parse(raw)
|
|
54
|
+
} catch {
|
|
55
|
+
throw new ValidationError('The bundle manifest is not valid JSON.')
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
const manifest = parsed as Partial<BackupManifest>
|
|
59
|
+
if (manifest.format !== 1) {
|
|
60
|
+
throw new ValidationError(
|
|
61
|
+
`This bundle declares format ${JSON.stringify(manifest.format)}; this build restores format 1.`,
|
|
62
|
+
)
|
|
63
|
+
}
|
|
64
|
+
if (manifest.uploads !== 'included' && manifest.uploads !== 'skipped') {
|
|
65
|
+
throw new ValidationError('The bundle manifest does not say whether uploads are included.')
|
|
66
|
+
}
|
|
67
|
+
if (typeof manifest.createdAt !== 'string' || typeof manifest.version !== 'string') {
|
|
68
|
+
throw new ValidationError('The bundle manifest is missing createdAt or version.')
|
|
69
|
+
}
|
|
70
|
+
if (manifest.filestore !== 'local' && manifest.filestore !== 's3') {
|
|
71
|
+
throw new ValidationError('The bundle manifest does not name a known file driver.')
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
return {
|
|
75
|
+
format: 1,
|
|
76
|
+
createdAt: manifest.createdAt,
|
|
77
|
+
version: manifest.version,
|
|
78
|
+
filestore: manifest.filestore,
|
|
79
|
+
uploads: manifest.uploads,
|
|
80
|
+
...(typeof manifest.bucket === 'string' ? { bucket: manifest.bucket } : {}),
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
const CONTENT_TYPES: ReadonlyMap<string, string> = new Map([
|
|
85
|
+
['.avif', 'image/avif'],
|
|
86
|
+
['.gif', 'image/gif'],
|
|
87
|
+
['.jpeg', 'image/jpeg'],
|
|
88
|
+
['.jpg', 'image/jpeg'],
|
|
89
|
+
['.png', 'image/png'],
|
|
90
|
+
['.svg', 'image/svg+xml'],
|
|
91
|
+
['.webp', 'image/webp'],
|
|
92
|
+
])
|
|
93
|
+
|
|
94
|
+
export function contentTypeFor(key: string): string {
|
|
95
|
+
return CONTENT_TYPES.get(path.extname(key).toLowerCase()) ?? 'application/octet-stream'
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
export function formatBytes(size: number): string {
|
|
99
|
+
let value = size
|
|
100
|
+
let unit = 'B'
|
|
101
|
+
for (const next of ['KiB', 'MiB', 'GiB', 'TiB']) {
|
|
102
|
+
if (value < 1024) break
|
|
103
|
+
value /= 1024
|
|
104
|
+
unit = next
|
|
105
|
+
}
|
|
106
|
+
return unit === 'B' ? `${value} B` : `${value.toFixed(value >= 10 ? 0 : 1)} ${unit}`
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
export interface RestoreLimits {
|
|
110
|
+
readonly archiveBytes: number
|
|
111
|
+
readonly members: number
|
|
112
|
+
readonly memberBytes: number
|
|
113
|
+
readonly expandedBytes: number
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
const RESTORE_LIMIT_DEFAULTS: RestoreLimits = {
|
|
117
|
+
archiveBytes: 2 * 1024 * 1024 * 1024,
|
|
118
|
+
members: 100_000,
|
|
119
|
+
memberBytes: 1024 * 1024 * 1024,
|
|
120
|
+
expandedBytes: 8 * 1024 * 1024 * 1024,
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function positiveInteger(value: string | undefined, variable: string, fallback: number): number {
|
|
124
|
+
if (value === undefined || value === '') return fallback
|
|
125
|
+
if (!/^\d+$/.test(value)) {
|
|
126
|
+
throw new ValidationError(`${variable} must be a positive integer number of bytes or members.`)
|
|
127
|
+
}
|
|
128
|
+
const parsed = Number(value)
|
|
129
|
+
if (!Number.isSafeInteger(parsed) || parsed < 1) {
|
|
130
|
+
throw new ValidationError(`${variable} must be a positive integer number of bytes or members.`)
|
|
131
|
+
}
|
|
132
|
+
return parsed
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
export function restoreLimits(environment: NodeJS.ProcessEnv): RestoreLimits {
|
|
136
|
+
return {
|
|
137
|
+
archiveBytes: positiveInteger(
|
|
138
|
+
environment.MEITH_RESTORE_MAX_ARCHIVE_BYTES,
|
|
139
|
+
'MEITH_RESTORE_MAX_ARCHIVE_BYTES',
|
|
140
|
+
RESTORE_LIMIT_DEFAULTS.archiveBytes,
|
|
141
|
+
),
|
|
142
|
+
members: positiveInteger(
|
|
143
|
+
environment.MEITH_RESTORE_MAX_MEMBERS,
|
|
144
|
+
'MEITH_RESTORE_MAX_MEMBERS',
|
|
145
|
+
RESTORE_LIMIT_DEFAULTS.members,
|
|
146
|
+
),
|
|
147
|
+
memberBytes: positiveInteger(
|
|
148
|
+
environment.MEITH_RESTORE_MAX_MEMBER_BYTES,
|
|
149
|
+
'MEITH_RESTORE_MAX_MEMBER_BYTES',
|
|
150
|
+
RESTORE_LIMIT_DEFAULTS.memberBytes,
|
|
151
|
+
),
|
|
152
|
+
expandedBytes: positiveInteger(
|
|
153
|
+
environment.MEITH_RESTORE_MAX_EXPANDED_BYTES,
|
|
154
|
+
'MEITH_RESTORE_MAX_EXPANDED_BYTES',
|
|
155
|
+
RESTORE_LIMIT_DEFAULTS.expandedBytes,
|
|
156
|
+
),
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
interface ArchiveMember {
|
|
161
|
+
readonly name: string
|
|
162
|
+
readonly type: string
|
|
163
|
+
readonly size: number
|
|
164
|
+
}
|
|
165
|
+
|
|
166
|
+
function normalizedArchiveName(name: string): string | undefined {
|
|
167
|
+
if (name === '' || name.includes('\\') || name.includes('\0')) return undefined
|
|
168
|
+
const withoutDirectoryMarker = name.endsWith('/') ? name.slice(0, -1) : name
|
|
169
|
+
if (withoutDirectoryMarker === '.') return '.'
|
|
170
|
+
const normalized = withoutDirectoryMarker.replace(/^\.\//, '')
|
|
171
|
+
if (normalized === '' || path.posix.isAbsolute(normalized)) return undefined
|
|
172
|
+
const parts = normalized.split('/')
|
|
173
|
+
if (parts.some((part) => part === '' || part === '.' || part === '..')) return undefined
|
|
174
|
+
return normalized
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
export function validateArchiveListing(
|
|
178
|
+
namesOutput: string,
|
|
179
|
+
verboseOutput: string,
|
|
180
|
+
limits: RestoreLimits,
|
|
181
|
+
allowedTypes: ReadonlySet<string>,
|
|
182
|
+
): readonly ArchiveMember[] {
|
|
183
|
+
const names = namesOutput === '' ? [] : namesOutput.replace(/\n$/, '').split('\n')
|
|
184
|
+
const verbose = verboseOutput === '' ? [] : verboseOutput.replace(/\n$/, '').split('\n')
|
|
185
|
+
if (names.length !== verbose.length) {
|
|
186
|
+
throw new ValidationError('The archive has malformed member names.')
|
|
187
|
+
}
|
|
188
|
+
if (names.length > limits.members) {
|
|
189
|
+
throw new ValidationError(`The archive has more than ${limits.members} members.`)
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
const seen = new Set<string>()
|
|
193
|
+
let expandedBytes = 0
|
|
194
|
+
return names.map((rawName, index) => {
|
|
195
|
+
const name = normalizedArchiveName(rawName)
|
|
196
|
+
if (name === undefined || seen.has(name)) {
|
|
197
|
+
throw new ValidationError(`The archive contains an unsafe or duplicate member: ${rawName}`)
|
|
198
|
+
}
|
|
199
|
+
seen.add(name)
|
|
200
|
+
|
|
201
|
+
const fields = verbose[index]?.trim().split(/\s+/) ?? []
|
|
202
|
+
const type = fields[0]?.[0] ?? ''
|
|
203
|
+
const size = Number(fields[2])
|
|
204
|
+
if (!allowedTypes.has(type) || !Number.isSafeInteger(size) || size < 0) {
|
|
205
|
+
throw new ValidationError(`The archive contains an unsupported member: ${name}`)
|
|
206
|
+
}
|
|
207
|
+
if (size > limits.memberBytes) {
|
|
208
|
+
throw new ValidationError(`The archive member ${name} exceeds the per-member size limit.`)
|
|
209
|
+
}
|
|
210
|
+
expandedBytes += size
|
|
211
|
+
if (!Number.isSafeInteger(expandedBytes) || expandedBytes > limits.expandedBytes) {
|
|
212
|
+
throw new ValidationError('The archive exceeds the expanded-size limit.')
|
|
213
|
+
}
|
|
214
|
+
return { name, type, size }
|
|
215
|
+
})
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
async function inspectArchive(
|
|
219
|
+
archive: string,
|
|
220
|
+
limits: RestoreLimits,
|
|
221
|
+
allowedTypes: ReadonlySet<string>,
|
|
222
|
+
): Promise<readonly ArchiveMember[]> {
|
|
223
|
+
const [names, verbose] = await Promise.all([
|
|
224
|
+
run('tar', ['tzf', archive]),
|
|
225
|
+
run('tar', ['tvzf', archive]),
|
|
226
|
+
])
|
|
227
|
+
return validateArchiveListing(names, verbose, limits, allowedTypes)
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
function missingToolError(command: string): ConfigurationError {
|
|
231
|
+
return new ConfigurationError(
|
|
232
|
+
`${command} was not found on PATH. The shipped image carries the postgres client ` +
|
|
233
|
+
'tools; elsewhere install them (postgresql18-client on Alpine, ' +
|
|
234
|
+
'postgresql-client on Debian and Ubuntu).',
|
|
235
|
+
)
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
const POSTGRES_PARAMETERS: Readonly<Record<string, string>> = {
|
|
239
|
+
application_name: 'PGAPPNAME',
|
|
240
|
+
channel_binding: 'PGCHANNELBINDING',
|
|
241
|
+
connect_timeout: 'PGCONNECT_TIMEOUT',
|
|
242
|
+
gssencmode: 'PGGSSENCMODE',
|
|
243
|
+
options: 'PGOPTIONS',
|
|
244
|
+
requirepeer: 'PGREQUIREPEER',
|
|
245
|
+
sslcert: 'PGSSLCERT',
|
|
246
|
+
sslcompression: 'PGSSLCOMPRESSION',
|
|
247
|
+
sslcrl: 'PGSSLCRL',
|
|
248
|
+
sslcrldir: 'PGSSLCRLDIR',
|
|
249
|
+
sslkey: 'PGSSLKEY',
|
|
250
|
+
sslmode: 'PGSSLMODE',
|
|
251
|
+
sslpassword: 'PGSSLPASSWORD',
|
|
252
|
+
sslrootcert: 'PGSSLROOTCERT',
|
|
253
|
+
target_session_attrs: 'PGTARGETSESSIONATTRS',
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
export function postgresClientEnvironment(
|
|
257
|
+
connectionString: string,
|
|
258
|
+
variable: string,
|
|
259
|
+
): NodeJS.ProcessEnv {
|
|
260
|
+
let url: URL
|
|
261
|
+
try {
|
|
262
|
+
url = new URL(connectionString)
|
|
263
|
+
} catch {
|
|
264
|
+
throw new ValidationError(`${variable} must be a valid postgres:// connection string.`)
|
|
265
|
+
}
|
|
266
|
+
|
|
267
|
+
if (url.protocol !== 'postgres:' && url.protocol !== 'postgresql:') {
|
|
268
|
+
throw new ValidationError(`${variable} must be a postgres:// connection string.`)
|
|
269
|
+
}
|
|
270
|
+
let database: string
|
|
271
|
+
let username: string
|
|
272
|
+
let password: string
|
|
273
|
+
try {
|
|
274
|
+
database = decodeURIComponent(url.pathname.replace(/^\//, ''))
|
|
275
|
+
username = decodeURIComponent(url.username)
|
|
276
|
+
password = decodeURIComponent(url.password)
|
|
277
|
+
} catch {
|
|
278
|
+
throw new ValidationError(`${variable} contains invalid percent-encoding.`)
|
|
279
|
+
}
|
|
280
|
+
if (url.hostname === '' || username === '' || database === '') {
|
|
281
|
+
throw new ValidationError(`${variable} must include a host, user, and database name.`)
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
const childEnv: NodeJS.ProcessEnv = { ...process.env }
|
|
285
|
+
for (const environmentVariable of [
|
|
286
|
+
'PGAPPNAME',
|
|
287
|
+
'PGCHANNELBINDING',
|
|
288
|
+
'PGCONNECT_TIMEOUT',
|
|
289
|
+
'PGDATABASE',
|
|
290
|
+
'PGGSSENCMODE',
|
|
291
|
+
'PGHOST',
|
|
292
|
+
'PGHOSTADDR',
|
|
293
|
+
'PGOPTIONS',
|
|
294
|
+
'PGPASSWORD',
|
|
295
|
+
'PGPORT',
|
|
296
|
+
'PGREQUIREPEER',
|
|
297
|
+
'PGSERVICE',
|
|
298
|
+
'PGSERVICEFILE',
|
|
299
|
+
'PGSSLCERT',
|
|
300
|
+
'PGSSLCOMPRESSION',
|
|
301
|
+
'PGSSLCRL',
|
|
302
|
+
'PGSSLCRLDIR',
|
|
303
|
+
'PGSSLKEY',
|
|
304
|
+
'PGSSLMODE',
|
|
305
|
+
'PGSSLPASSWORD',
|
|
306
|
+
'PGSSLROOTCERT',
|
|
307
|
+
'PGTARGETSESSIONATTRS',
|
|
308
|
+
'PGUSER',
|
|
309
|
+
]) {
|
|
310
|
+
delete childEnv[environmentVariable]
|
|
311
|
+
}
|
|
312
|
+
Object.assign(childEnv, {
|
|
313
|
+
PGHOST: url.hostname.replace(/^\[|\]$/g, ''),
|
|
314
|
+
PGPORT: url.port || '5432',
|
|
315
|
+
PGUSER: username,
|
|
316
|
+
PGDATABASE: database,
|
|
317
|
+
})
|
|
318
|
+
if (password !== '') childEnv.PGPASSWORD = password
|
|
319
|
+
|
|
320
|
+
for (const [parameter, environmentVariable] of Object.entries(POSTGRES_PARAMETERS)) {
|
|
321
|
+
const value = url.searchParams.get(parameter)
|
|
322
|
+
if (value !== null) childEnv[environmentVariable] = value
|
|
323
|
+
}
|
|
324
|
+
return childEnv
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
async function run(
|
|
328
|
+
command: string,
|
|
329
|
+
args: readonly string[],
|
|
330
|
+
childEnv: NodeJS.ProcessEnv = process.env,
|
|
331
|
+
): Promise<string> {
|
|
332
|
+
return new Promise<string>((resolvePromise, reject) => {
|
|
333
|
+
const child = spawn(command, args, {
|
|
334
|
+
env: childEnv,
|
|
335
|
+
stdio: ['ignore', 'pipe', 'pipe'],
|
|
336
|
+
})
|
|
337
|
+
let stdout = ''
|
|
338
|
+
let stderr = ''
|
|
339
|
+
child.stdout.on('data', (chunk: Buffer) => {
|
|
340
|
+
stdout += String(chunk)
|
|
341
|
+
})
|
|
342
|
+
child.stderr.on('data', (chunk: Buffer) => {
|
|
343
|
+
stderr += String(chunk)
|
|
344
|
+
})
|
|
345
|
+
child.on('error', (error) => {
|
|
346
|
+
reject((error as NodeJS.ErrnoException).code === 'ENOENT' ? missingToolError(command) : error)
|
|
347
|
+
})
|
|
348
|
+
child.on('close', (code) => {
|
|
349
|
+
if (code === 0) resolvePromise(stdout)
|
|
350
|
+
else {
|
|
351
|
+
reject(
|
|
352
|
+
new ConfigurationError(
|
|
353
|
+
`${command} exited with ${code === null ? 'a signal' : `code ${code}`}.` +
|
|
354
|
+
(stderr.trim() === '' ? '' : `\n${stderr.trim()}`),
|
|
355
|
+
),
|
|
356
|
+
)
|
|
357
|
+
}
|
|
358
|
+
})
|
|
359
|
+
})
|
|
360
|
+
}
|
|
361
|
+
|
|
362
|
+
export async function reserveBackupDestination(destination: string): Promise<void> {
|
|
363
|
+
const file = await open(destination, 'wx', 0o600)
|
|
364
|
+
await file.close()
|
|
365
|
+
}
|
|
366
|
+
|
|
367
|
+
async function stageLocalUploads(stage: string): Promise<'included' | 'skipped'> {
|
|
368
|
+
const exists = await stat(env.UPLOADS_DIR).then(
|
|
369
|
+
(info) => info.isDirectory(),
|
|
370
|
+
() => false,
|
|
371
|
+
)
|
|
372
|
+
if (!exists) {
|
|
373
|
+
console.log(`No uploads directory at ${env.UPLOADS_DIR}; the bundle carries none.`)
|
|
374
|
+
return 'skipped'
|
|
375
|
+
}
|
|
376
|
+
|
|
377
|
+
await run('tar', ['czf', path.join(stage, 'uploads.tar.gz'), '-C', env.UPLOADS_DIR, '.'])
|
|
378
|
+
return 'included'
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
async function stageS3Uploads(stage: string): Promise<'included' | 'skipped'> {
|
|
382
|
+
const store = S3FileStore.fromEnv(env)
|
|
383
|
+
const dir = path.join(stage, 'uploads')
|
|
384
|
+
await mkdir(dir, { recursive: true })
|
|
385
|
+
|
|
386
|
+
let pulled = 0
|
|
387
|
+
for await (const key of store.listKeys()) {
|
|
388
|
+
const target = path.resolve(dir, key)
|
|
389
|
+
if (target !== dir && !target.startsWith(dir + path.sep)) {
|
|
390
|
+
console.warn(`Skipping object with an unsafe key: ${key}`)
|
|
391
|
+
continue
|
|
392
|
+
}
|
|
393
|
+
const body = await store.get(key)
|
|
394
|
+
if (body === undefined) continue
|
|
395
|
+
await mkdir(path.dirname(target), { recursive: true })
|
|
396
|
+
await writeFile(target, body)
|
|
397
|
+
pulled++
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
if (pulled === 0) {
|
|
401
|
+
console.log(`The ${env.S3_BUCKET} bucket is empty; the bundle carries no uploads.`)
|
|
402
|
+
await rm(dir, { recursive: true, force: true })
|
|
403
|
+
return 'skipped'
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
await run('tar', ['czf', path.join(stage, 'uploads.tar.gz'), '-C', dir, '.'])
|
|
407
|
+
await rm(dir, { recursive: true, force: true })
|
|
408
|
+
console.log(`Pulled ${pulled} object(s) from ${env.S3_BUCKET}.`)
|
|
409
|
+
return 'included'
|
|
410
|
+
}
|
|
411
|
+
|
|
412
|
+
export async function backupCommand(args: readonly string[]): Promise<number> {
|
|
413
|
+
const { flags } = parseFlags(args)
|
|
414
|
+
requirePostgres()
|
|
415
|
+
|
|
416
|
+
const mode = resolveUploadsMode(env.FILESTORE_DRIVER, optional(flags, 'uploads'))
|
|
417
|
+
const now = new Date()
|
|
418
|
+
const out = path.resolve(optional(flags, 'out') ?? bundleName(now))
|
|
419
|
+
|
|
420
|
+
const stage = await mkdtemp(path.join(tmpdir(), 'meith-backup-'))
|
|
421
|
+
let destinationCreated = false
|
|
422
|
+
try {
|
|
423
|
+
console.log(
|
|
424
|
+
env.DIRECT_DATABASE_URL === undefined
|
|
425
|
+
? 'Dumping the database…'
|
|
426
|
+
: 'Dumping the database over DIRECT_DATABASE_URL…',
|
|
427
|
+
)
|
|
428
|
+
const databaseVariable =
|
|
429
|
+
env.DIRECT_DATABASE_URL === undefined ? 'DATABASE_URL' : 'DIRECT_DATABASE_URL'
|
|
430
|
+
const databaseEnvironment = postgresClientEnvironment(migrationUrl(env), databaseVariable)
|
|
431
|
+
await run(
|
|
432
|
+
'pg_dump',
|
|
433
|
+
['--format=custom', '--no-owner', '--no-privileges', '--file', path.join(stage, 'db.dump')],
|
|
434
|
+
databaseEnvironment,
|
|
435
|
+
)
|
|
436
|
+
|
|
437
|
+
const uploads =
|
|
438
|
+
mode === 'skip'
|
|
439
|
+
? 'skipped'
|
|
440
|
+
: env.FILESTORE_DRIVER === 's3'
|
|
441
|
+
? await stageS3Uploads(stage)
|
|
442
|
+
: await stageLocalUploads(stage)
|
|
443
|
+
|
|
444
|
+
const manifest: BackupManifest = {
|
|
445
|
+
format: 1,
|
|
446
|
+
createdAt: now.toISOString(),
|
|
447
|
+
version: CODE_VERSION,
|
|
448
|
+
filestore: env.FILESTORE_DRIVER,
|
|
449
|
+
uploads,
|
|
450
|
+
...(env.S3_BUCKET === undefined ? {} : { bucket: env.S3_BUCKET }),
|
|
451
|
+
}
|
|
452
|
+
await writeFile(path.join(stage, 'manifest.json'), `${JSON.stringify(manifest, null, 2)}\n`)
|
|
453
|
+
|
|
454
|
+
const members = ['manifest.json', 'db.dump']
|
|
455
|
+
if (uploads === 'included') members.push('uploads.tar.gz')
|
|
456
|
+
await reserveBackupDestination(out)
|
|
457
|
+
destinationCreated = true
|
|
458
|
+
await run('tar', ['czf', out, '-C', stage, ...members])
|
|
459
|
+
await chmod(out, 0o600)
|
|
460
|
+
|
|
461
|
+
const size = (await stat(out)).size
|
|
462
|
+
console.log(
|
|
463
|
+
`Wrote ${out} (${formatBytes(size)}): the database dump${
|
|
464
|
+
uploads === 'included' ? ' and the uploads' : ', no uploads'
|
|
465
|
+
}.`,
|
|
466
|
+
)
|
|
467
|
+
if (uploads === 'skipped' && env.FILESTORE_DRIVER === 's3' && mode !== 'include') {
|
|
468
|
+
console.log(
|
|
469
|
+
'The S3 bucket was not pulled — it has its own backup story. ' +
|
|
470
|
+
'Run with --uploads include for a bundle that carries every object.',
|
|
471
|
+
)
|
|
472
|
+
}
|
|
473
|
+
if (uploads === 'skipped' && mode === 'skip') {
|
|
474
|
+
console.log('Restoring this bundle gives a board whose posts have broken images.')
|
|
475
|
+
}
|
|
476
|
+
console.log(
|
|
477
|
+
'Copy the bundle off this machine: a backup on the server is a backup of the ' +
|
|
478
|
+
'thing most likely to fail.',
|
|
479
|
+
)
|
|
480
|
+
destinationCreated = false
|
|
481
|
+
return 0
|
|
482
|
+
} finally {
|
|
483
|
+
if (destinationCreated) await rm(out, { force: true })
|
|
484
|
+
await rm(stage, { recursive: true, force: true })
|
|
485
|
+
}
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
async function walk(dir: string): Promise<readonly string[]> {
|
|
489
|
+
const files: string[] = []
|
|
490
|
+
for (const entry of await readdir(dir, { withFileTypes: true })) {
|
|
491
|
+
const full = path.join(dir, entry.name)
|
|
492
|
+
if (entry.isDirectory()) files.push(...(await walk(full)))
|
|
493
|
+
else if (entry.isFile()) files.push(full)
|
|
494
|
+
}
|
|
495
|
+
return files
|
|
496
|
+
}
|
|
497
|
+
|
|
498
|
+
async function validateUploadsArchive(stage: string, limits: RestoreLimits): Promise<void> {
|
|
499
|
+
const members = await inspectArchive(
|
|
500
|
+
path.join(stage, 'uploads.tar.gz'),
|
|
501
|
+
limits,
|
|
502
|
+
new Set(['-', 'd']),
|
|
503
|
+
)
|
|
504
|
+
for (const member of members) {
|
|
505
|
+
if (member.name === '.' && member.type !== 'd') {
|
|
506
|
+
throw new ValidationError('The uploads archive root is not a directory.')
|
|
507
|
+
}
|
|
508
|
+
}
|
|
509
|
+
}
|
|
510
|
+
|
|
511
|
+
async function extractUploads(stage: string, dir: string, limits: RestoreLimits): Promise<void> {
|
|
512
|
+
await validateUploadsArchive(stage, limits)
|
|
513
|
+
const existing = await readdir(dir).catch((error: NodeJS.ErrnoException) => {
|
|
514
|
+
if (error.code === 'ENOENT') return undefined
|
|
515
|
+
throw error
|
|
516
|
+
})
|
|
517
|
+
if (existing !== undefined && existing.length > 0) {
|
|
518
|
+
throw new ValidationError(
|
|
519
|
+
`${dir} is not empty. Restore the uploads into a fresh directory (--uploads-dir), ` +
|
|
520
|
+
'the same way the database goes into a fresh database.',
|
|
521
|
+
)
|
|
522
|
+
}
|
|
523
|
+
|
|
524
|
+
await mkdir(dir, { recursive: true })
|
|
525
|
+
await run('tar', ['xzf', path.join(stage, 'uploads.tar.gz'), '-C', dir])
|
|
526
|
+
console.log(`Restored the uploads into ${dir}.`)
|
|
527
|
+
}
|
|
528
|
+
|
|
529
|
+
async function pushUploadsToS3(stage: string, limits: RestoreLimits): Promise<void> {
|
|
530
|
+
await validateUploadsArchive(stage, limits)
|
|
531
|
+
const dir = path.join(stage, 'uploads-extract')
|
|
532
|
+
await mkdir(dir, { recursive: true })
|
|
533
|
+
await run('tar', ['xzf', path.join(stage, 'uploads.tar.gz'), '-C', dir])
|
|
534
|
+
|
|
535
|
+
const store = S3FileStore.fromEnv(env)
|
|
536
|
+
let pushed = 0
|
|
537
|
+
for (const file of await walk(dir)) {
|
|
538
|
+
const key = path.relative(dir, file).split(path.sep).join('/')
|
|
539
|
+
await store.put(key, await readFile(file), {
|
|
540
|
+
contentType: contentTypeFor(key),
|
|
541
|
+
visibility: 'public',
|
|
542
|
+
})
|
|
543
|
+
pushed++
|
|
544
|
+
}
|
|
545
|
+
console.log(`Uploaded ${pushed} object(s) to ${env.S3_BUCKET}.`)
|
|
546
|
+
}
|
|
547
|
+
|
|
548
|
+
const RESTORE_USAGE =
|
|
549
|
+
'Usage: RESTORE_DATABASE_URL=<postgres://…> community restore <bundle.tar.gz> ' +
|
|
550
|
+
'[--uploads-dir <dir>] [--skip-uploads]'
|
|
551
|
+
|
|
552
|
+
export function restoreDatabaseUrl(
|
|
553
|
+
args: readonly string[],
|
|
554
|
+
environment: NodeJS.ProcessEnv,
|
|
555
|
+
): string {
|
|
556
|
+
const { flags } = parseFlags(args)
|
|
557
|
+
if (flags.has('database-url')) {
|
|
558
|
+
throw new ValidationError(
|
|
559
|
+
'--database-url is not supported because process arguments are observable. ' +
|
|
560
|
+
'Set RESTORE_DATABASE_URL in the environment instead.',
|
|
561
|
+
)
|
|
562
|
+
}
|
|
563
|
+
const target = environment.RESTORE_DATABASE_URL
|
|
564
|
+
if (target === undefined || target === '') {
|
|
565
|
+
throw new ValidationError(`RESTORE_DATABASE_URL is required.\n${RESTORE_USAGE}`)
|
|
566
|
+
}
|
|
567
|
+
return target
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
export async function restoreCommand(args: readonly string[]): Promise<number> {
|
|
571
|
+
const { flags, positional } = parseFlags(args)
|
|
572
|
+
|
|
573
|
+
const bundle = positional[0]
|
|
574
|
+
if (bundle === undefined) throw new ValidationError(RESTORE_USAGE)
|
|
575
|
+
|
|
576
|
+
const target = restoreDatabaseUrl(args, process.env)
|
|
577
|
+
const databaseEnvironment = postgresClientEnvironment(target, 'RESTORE_DATABASE_URL')
|
|
578
|
+
|
|
579
|
+
const bundleInfo = await stat(bundle).catch(() => undefined)
|
|
580
|
+
if (bundleInfo === undefined || !bundleInfo.isFile()) {
|
|
581
|
+
throw new ValidationError(`No such bundle: ${bundle}`)
|
|
582
|
+
}
|
|
583
|
+
|
|
584
|
+
const limits = restoreLimits(process.env)
|
|
585
|
+
if (bundleInfo.size > limits.archiveBytes) {
|
|
586
|
+
throw new ValidationError('The backup bundle exceeds MEITH_RESTORE_MAX_ARCHIVE_BYTES.')
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
const stage = await mkdtemp(path.join(tmpdir(), 'meith-restore-'))
|
|
590
|
+
try {
|
|
591
|
+
const stagedBundle = path.join(stage, 'bundle.tar.gz')
|
|
592
|
+
await copyFile(path.resolve(bundle), stagedBundle)
|
|
593
|
+
const members = await inspectArchive(stagedBundle, limits, new Set(['-']))
|
|
594
|
+
const possibleMembers = new Set(['manifest.json', 'db.dump', 'uploads.tar.gz'])
|
|
595
|
+
if (members.some((member) => !possibleMembers.has(member.name))) {
|
|
596
|
+
throw new ValidationError('The backup bundle contains an unexpected member.')
|
|
597
|
+
}
|
|
598
|
+
await run('tar', ['xzf', stagedBundle, '-C', stage, 'manifest.json'])
|
|
599
|
+
const manifest = parseManifest(await readFile(path.join(stage, 'manifest.json'), 'utf8'))
|
|
600
|
+
const expectedMembers = new Set(['manifest.json', 'db.dump'])
|
|
601
|
+
if (manifest.uploads === 'included') expectedMembers.add('uploads.tar.gz')
|
|
602
|
+
if (
|
|
603
|
+
members.length !== expectedMembers.size ||
|
|
604
|
+
members.some((member) => !expectedMembers.has(member.name))
|
|
605
|
+
) {
|
|
606
|
+
throw new ValidationError('The backup bundle members do not match its manifest.')
|
|
607
|
+
}
|
|
608
|
+
const restoreMembers = ['db.dump']
|
|
609
|
+
if (manifest.uploads === 'included') restoreMembers.push('uploads.tar.gz')
|
|
610
|
+
await run('tar', ['xzf', stagedBundle, '-C', stage, ...restoreMembers])
|
|
611
|
+
|
|
612
|
+
const tables = (
|
|
613
|
+
await run(
|
|
614
|
+
'psql',
|
|
615
|
+
['-tAc', "select count(*) from information_schema.tables where table_schema = 'public'"],
|
|
616
|
+
databaseEnvironment,
|
|
617
|
+
)
|
|
618
|
+
).trim()
|
|
619
|
+
if (tables !== '0') {
|
|
620
|
+
throw new ValidationError(
|
|
621
|
+
`The target database already holds ${tables} table(s). Restore into a new, ` +
|
|
622
|
+
'empty database — a restore over a live board is how a bad backup becomes ' +
|
|
623
|
+
'two lost boards.',
|
|
624
|
+
)
|
|
625
|
+
}
|
|
626
|
+
|
|
627
|
+
console.log(`Restoring the backup taken ${manifest.createdAt} (version ${manifest.version})…`)
|
|
628
|
+
await run(
|
|
629
|
+
'pg_restore',
|
|
630
|
+
[
|
|
631
|
+
'--no-owner',
|
|
632
|
+
'--no-privileges',
|
|
633
|
+
'--dbname',
|
|
634
|
+
databaseEnvironment.PGDATABASE ?? '',
|
|
635
|
+
path.join(stage, 'db.dump'),
|
|
636
|
+
],
|
|
637
|
+
databaseEnvironment,
|
|
638
|
+
)
|
|
639
|
+
|
|
640
|
+
const applied = await runMigrations({ url: target })
|
|
641
|
+
console.log(
|
|
642
|
+
applied === 0
|
|
643
|
+
? 'Migrations: nothing to do — the dump matches this build.'
|
|
644
|
+
: `Migrations: applied ${applied} migration(s) the dump predates.`,
|
|
645
|
+
)
|
|
646
|
+
|
|
647
|
+
const posts = (
|
|
648
|
+
await run('psql', ['-tAc', 'select count(*) from posts'], databaseEnvironment)
|
|
649
|
+
).trim()
|
|
650
|
+
console.log(`The restored board holds ${posts} post(s).`)
|
|
651
|
+
|
|
652
|
+
if (manifest.uploads === 'included' && flags.get('skip-uploads') !== 'true') {
|
|
653
|
+
const uploadsDir = optional(flags, 'uploads-dir')
|
|
654
|
+
if (env.FILESTORE_DRIVER === 's3' && uploadsDir === undefined) {
|
|
655
|
+
await pushUploadsToS3(stage, limits)
|
|
656
|
+
} else {
|
|
657
|
+
await extractUploads(stage, uploadsDir ?? env.UPLOADS_DIR, limits)
|
|
658
|
+
}
|
|
659
|
+
} else if (manifest.uploads === 'skipped') {
|
|
660
|
+
console.log(
|
|
661
|
+
manifest.filestore === 's3'
|
|
662
|
+
? `This bundle carries no uploads — they live in the S3 bucket${
|
|
663
|
+
manifest.bucket === undefined ? '' : ` (${manifest.bucket})`
|
|
664
|
+
}.`
|
|
665
|
+
: 'This bundle carries no uploads.',
|
|
666
|
+
)
|
|
667
|
+
}
|
|
668
|
+
|
|
669
|
+
console.log(
|
|
670
|
+
'Point a staging deployment at the restored database, sign in as an ' +
|
|
671
|
+
'administrator, and open a thread with attachments before trusting it.',
|
|
672
|
+
)
|
|
673
|
+
return 0
|
|
674
|
+
} finally {
|
|
675
|
+
await rm(stage, { recursive: true, force: true })
|
|
676
|
+
}
|
|
677
|
+
}
|