@meith/cli 0.16.0

package/src/import-files.ts

@@ -0,0 +1,189 @@
+import { readdir, readFile } from 'node:fs/promises'
+import { basename, dirname, join, normalize, resolve, sep } from 'node:path'
+
+import { type AttachmentType, type ImageProcessor, sniff } from '@meith/attachments'
+import { AVATAR_BOX } from '@meith/avatars'
+import type { FileStore } from '@meith/core'
+import type { CopiedAttachment, CopiedAvatar, ImportFileCopier } from '@meith/db'
+import { imageProcessor } from '@meith/drivers/images'
+
+const EXTENSIONS: Readonly<Record<string, string>> = {
+  'image/png': '.png',
+  'image/jpeg': '.jpg',
+  'application/pdf': '.pdf',
+  'application/zip': '.zip',
+}
+
+export class UploadsDirCopier implements ImportFileCopier {
+  private readonly root: string
+
+  constructor(
+    root: string,
+    private readonly files: FileStore,
+    private readonly images: ImageProcessor = imageProcessor,
+  ) {
+    this.root = resolve(root)
+  }
+
+  async attachment(input: {
+    readonly legacyId: number
+    readonly path: string
+    readonly thumbnailPath: string | null
+    readonly filename: string
+    readonly contentType: string | null
+  }): Promise<CopiedAttachment> {
+    const bytes = await this.#read(input.path)
+    if (bytes === null) {
+      return { outcome: 'missing', reason: `file ${input.path} missing from uploads directory` }
+    }
+
+    const type = sniff(bytes)
+    if (type === undefined) {
+      return {
+        outcome: 'failed',
+        reason: `file ${input.path} is not a type this board accepts (png, jpeg, pdf, zip)`,
+      }
+    }
+
+    if (type.handling === 'opaque') {
+      const key = storageKey('attachments', input.legacyId, EXTENSIONS[type.contentType] ?? '')
+      await this.files.put(key, bytes, { contentType: type.contentType, visibility: 'private' })
+      return {
+        outcome: 'ready',
+        storageKey: key,
+        thumbnailKey: null,
+        contentType: type.contentType,
+        sizeBytes: bytes.byteLength,
+        width: null,
+        height: null,
+      }
+    }
+
+    return this.#reencode(input.path, input.legacyId, bytes, type)
+  }
+
+  async avatar(input: {
+    readonly legacyUserId: number
+    readonly path: string
+  }): Promise<CopiedAvatar> {
+    const bytes = await this.#read(input.path)
+    if (bytes === null) {
+      return { outcome: 'missing', reason: `avatar ${input.path} missing from uploads directory` }
+    }
+
+    const type = sniff(bytes)
+    if (type === undefined || type.codec === null) {
+      return { outcome: 'failed', reason: `avatar ${input.path} is not a png or jpeg` }
+    }
+
+    try {
+      const processed = await this.images.process({
+        bytes,
+        codec: type.codec,
+        fit: AVATAR_BOX,
+        thumbnail: false,
+      })
+      const key = storageKey('avatars', input.legacyUserId, EXTENSIONS[processed.contentType] ?? '')
+      await this.files.put(key, processed.bytes, {
+        contentType: processed.contentType,
+        visibility: 'public',
+      })
+      return { outcome: 'ready', key, width: processed.width, height: processed.height }
+    } catch {
+      return { outcome: 'failed', reason: `avatar ${input.path} could not be decoded` }
+    }
+  }
+
+  async #reencode(
+    path: string,
+    legacyId: number,
+    bytes: Uint8Array,
+    type: AttachmentType,
+  ): Promise<CopiedAttachment> {
+    if (type.codec === null) {
+      return { outcome: 'failed', reason: `file ${path} could not be decoded` }
+    }
+
+    try {
+      const processed = await this.images.process({ bytes, codec: type.codec })
+      const key = storageKey('attachments', legacyId, EXTENSIONS[processed.contentType] ?? '')
+      await this.files.put(key, processed.bytes, {
+        contentType: processed.contentType,
+        visibility: 'private',
+      })
+
+      let thumbnailKey: string | null = null
+      if (processed.thumbnail !== undefined) {
+        thumbnailKey = storageKey('attachments', legacyId, '.thumb.jpg')
+        await this.files.put(thumbnailKey, processed.thumbnail.bytes, {
+          contentType: processed.thumbnail.contentType,
+          visibility: 'private',
+        })
+      }
+
+      return {
+        outcome: 'ready',
+        storageKey: key,
+        thumbnailKey,
+        contentType: processed.contentType,
+        sizeBytes: processed.bytes.byteLength,
+        width: processed.width,
+        height: processed.height,
+      }
+    } catch {
+      return { outcome: 'failed', reason: `file ${path} could not be decoded` }
+    }
+  }
+
+  async #read(relative: string): Promise<Uint8Array | null> {
+    const target = this.#within(relative)
+    if (target === null) return null
+
+    if (basename(target).includes('*')) return this.#readGlob(target)
+
+    try {
+      return await readFile(target)
+    } catch (error) {
+      if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null
+      throw error
+    }
+  }
+
+  async #readGlob(target: string): Promise<Uint8Array | null> {
+    const dir = dirname(target)
+    const pattern = basename(target)
+    const star = pattern.indexOf('*')
+    const prefix = pattern.slice(0, star)
+    const suffix = pattern.slice(star + 1)
+
+    let entries: readonly string[]
+    try {
+      entries = await readdir(dir)
+    } catch {
+      return null
+    }
+
+    const match = entries.find(
+      (entry) =>
+        entry.startsWith(prefix) && entry.endsWith(suffix) && entry.length >= pattern.length - 1,
+    )
+    if (match === undefined) return null
+
+    try {
+      return await readFile(join(dir, match))
+    } catch {
+      return null
+    }
+  }
+
+  #within(relative: string): string | null {
+    if (relative.includes('\0') || relative.trim() === '') return null
+    const target = resolve(this.root, normalize(relative))
+    if (target !== this.root && !target.startsWith(this.root + sep)) return null
+    return target
+  }
+}
+
+function storageKey(kind: string, legacyId: number, extension: string): string {
+  return `import/${kind}/${legacyId}${extension}`
+}

package/src/import.ts

@@ -0,0 +1,142 @@
+import {
+  currentImportRun,
+  finishImportRun,
+  getDb,
+  type ImportFileCopier,
+  type ImportSourceName,
+  PostgresImportSink,
+  saveImportProgress,
+  startImportRun,
+} from '@meith/db'
+import { drivers } from '@meith/drivers'
+import {
+  type Cursors,
+  type ImportReport,
+  MysqlMybbSource,
+  MysqlPhpbbSource,
+  NO_PROGRESS,
+  runImport,
+} from '@meith/import'
+
+import { integer, optional, parseFlags, required } from './args'
+import { UploadsDirCopier } from './import-files'
+
+interface ClosableSource {
+  close(): Promise<void>
+}
+
+export async function importCommand(args: readonly string[]): Promise<number> {
+  const { flags } = parseFlags(args)
+
+  const sourceName = (optional(flags, 'source') ?? 'mybb') as ImportSourceName
+  if (sourceName !== 'mybb' && sourceName !== 'phpbb') {
+    console.error(`Unknown import source "${sourceName}". Supported sources: mybb, phpbb.`)
+    return 1
+  }
+
+  const password = process.env.IMPORT_SOURCE_PASSWORD ?? process.env.MYBB_PASSWORD
+  if (password === undefined) {
+    console.error(
+      'Set IMPORT_SOURCE_PASSWORD (MYBB_PASSWORD is also accepted). It is read from the\n' +
+        'environment rather than a flag because a password in argv is in your shell\n' +
+        'history and in `ps` for every user on the box.',
+    )
+    return 1
+  }
+
+  const options = {
+    host: required(flags, 'host'),
+    port: integer(flags, 'port'),
+    user: required(flags, 'user'),
+    password,
+    database: required(flags, 'database'),
+    tablePrefix: optional(flags, 'prefix'),
+    charset: optional(flags, 'charset'),
+    ssl: flags.has('ssl'),
+  }
+
+  const source =
+    sourceName === 'mybb'
+      ? await MysqlMybbSource.connect(options)
+      : await MysqlPhpbbSource.connect(options)
+
+  try {
+    const db = getDb()
+
+    const uploadsDir = optional(flags, 'uploads-dir')
+    const copier: ImportFileCopier | undefined =
+      uploadsDir === undefined ? undefined : new UploadsDirCopier(uploadsDir, drivers().files)
+
+    const sink = new PostgresImportSink(db, { copier })
+
+    const existing = await currentImportRun(db, sourceName)
+    const from: Cursors = { ...NO_PROGRESS, ...existing?.cursors }
+    const runId = existing?.id ?? (await startImportRun(db, sourceName))
+
+    if (existing !== null) {
+      console.log(`Resuming import run ${runId} from ${describe(from)}.`)
+    }
+
+    const report = await runImport({
+      source,
+      sink,
+      pageSize: integer(flags, 'page-size'),
+      budget: integer(flags, 'budget'),
+      from,
+    })
+
+    await saveImportProgress(db, runId, report.cursors, report.readThisRun, report.kinds)
+    if (report.finished) await finishImportRun(db, runId, 'finished', null)
+
+    print(report, uploadsDir !== undefined)
+    return 0
+  } finally {
+    await (source as ClosableSource).close()
+  }
+}
+
+const describe = (cursors: Cursors): string =>
+  Object.entries(cursors)
+    .filter(([, id]) => id > 0)
+    .map(([kind, id]) => `${kind}:${id}`)
+    .join(' ')
+
+function print(report: ImportReport, copiedFiles: boolean): void {
+  const width = Math.max(...Object.keys(report.kinds).map((k) => k.length))
+
+  for (const [kind, result] of Object.entries(report.kinds)) {
+    console.log(
+      `  ${kind.padEnd(width)}  read ${String(result.read).padStart(7)}  ` +
+        `inserted ${String(result.inserted).padStart(7)}  ` +
+        `updated ${String(result.updated).padStart(7)}  ` +
+        `skipped ${String(result.skipped.length).padStart(5)}`,
+    )
+  }
+
+  for (const [kind, result] of Object.entries(report.kinds)) {
+    for (const skip of result.skipped.slice(0, 20)) {
+      console.log(`  skipped ${kind} ${skip.legacyId}: ${skip.reason}`)
+    }
+    if (result.skipped.length > 20) {
+      console.log(`  … and ${result.skipped.length - 20} more skipped ${kind}`)
+    }
+  }
+
+  console.log('')
+  if (report.finished) {
+    console.log(
+      'Import complete. Run `community task:run counters.reconcile` before opening the board.',
+    )
+    if (!copiedFiles) {
+      console.log(
+        'Attachment and avatar files were not copied (no --uploads-dir). ' +
+          'Run the same command again with --uploads-dir to bring the files across.',
+      )
+    }
+  } else {
+    console.log(
+      `Stopped after ${report.readThisRun.toLocaleString()} rows (the budget). ` +
+        'Not an error — run the same command again to continue from here.',
+    )
+  }
+}