@live-change/backup-service 0.9.207 → 0.9.209

package/backup.js

@@ -9,6 +9,7 @@ import { exec as cpExec } from 'child_process'
 const exec = util.promisify(cpExec)
 import dump from '@live-change/db-client/lib/dump.js'
 import { once } from 'events'
+import { finished } from 'node:stream/promises'
 import os from "os"
 import path from 'path'
 
@@ -17,6 +18,14 @@ function currentBackupPath(backupsDir = './backups/') {
   return `${backupsDir}/${dateString.substring(0, dateString.length - 1)}`
 }
 
+function createBackupLogger(backupsDir) {
+  const logPath = path.join(path.resolve(backupsDir), 'backups.log')
+  return async function log(message) {
+    const line = `${new Date().toISOString()} ${message}\n`
+    await fs.promises.appendFile(logPath, line, { encoding: 'utf8' })
+  }
+}
+
 async function writeDbBackup(stream) {
   let drainPromise
   async function write(object) {
@@ -27,11 +36,10 @@ async function writeDbBackup(stream) {
       drainPromise = null
     }
   }
-  await dump({
+  try {
+    await dump({
       dao: app.dao,
       db: app.databaseName,
-      //serverUrl: process.env.DB_URL || 'http://localhost:9417/api/ws',
-      //db: process.env.DB_NAME || 'test',
       structure: true,
       verbose: true,
       bucket: 128, // less database stress
@@ -39,64 +47,128 @@ async function writeDbBackup(stream) {
     },
     (method, ...args) => write({ type: 'request', method, parameters: args }),
       () => write({ type: 'sync' })
-  )
-  stream.end()
+    )
+    stream.end()
+    await finished(stream)
+  } catch (err) {
+    stream.destroy()
+    throw err
+  }
 }
 
-async function createBackup(backupPath = currentBackupPath()) {
-  await fs.promises.mkdir(backupPath)
-
-  const dbStream = fs.createWriteStream(path.resolve(backupPath, 'db.json'))
-  await writeDbBackup(dbStream)
-
-  const version = await (
-    fs.promises.readFile('./package.json', { encoding: 'utf8' })
-      .catch(e => 'unknown')
-      .then(data => JSON.parse(data).version)
-  )
-  const info = {
-    version: version,
-    hostname: os.hostname(),
-    directory: path.resolve('.')
-  }
+function parseBackupTimestamp(basename) {
+  const match = basename.match(/^(\d{4})_(\d{2})_(\d{2})_(\d{2})_(\d{2})_(\d{2})_(\d+)/)
+  if(!match) return null
+  const [_full, year, month, day, hour, minute, second, millis] = match
+  return new Date(`${year}-${month}-${day}T${hour}:${minute}:${second}.${millis}Z`)
+}
+
+async function createBackup(backupPath, log) {
+  const resolvedPath = path.resolve(backupPath)
+  const parentDir = path.dirname(resolvedPath)
+  const base = path.basename(resolvedPath)
+  const tarTmpPath = path.join(parentDir, `${base}.tar.gz.tmp`)
+
+  await log(`createBackup start backupPath=${resolvedPath}`)
+
+  try {
+    await fs.promises.mkdir(resolvedPath)
+    await log('mkdir done')
+
+    const dbStream = fs.createWriteStream(path.resolve(resolvedPath, 'db.json'))
+    await writeDbBackup(dbStream)
+    await log('db dump finished (stream closed)')
+
+    const version = await (
+      fs.promises.readFile('./package.json', { encoding: 'utf8' })
+        .catch(e => 'unknown')
+        .then(data => JSON.parse(data).version)
+    )
+    const info = {
+      version: version,
+      hostname: os.hostname(),
+      directory: path.resolve('.')
+    }
+
+    await fs.promises.writeFile(path.resolve(resolvedPath, 'info.json'), JSON.stringify(info, null, '  '))
+    await log('info.json written')
 
-  await fs.promises.writeFile(path.resolve(backupPath, 'info.json'), JSON.stringify(info, null, '  '))
+    await fse.copy("./storage", path.resolve(resolvedPath, "storage"))
+    await log('storage copied')
 
-  await fse.copy("./storage", path.resolve(backupPath, "storage"))
+    const command = `tar -zcf ../${base}.tar.gz.tmp storage info.json db.json`
+    console.log("EXEC TAR COMMAND:", command)
+    await log(`tar start command=${command}`)
+    await exec(command, { cwd: resolvedPath })
+    await log('tar done')
 
-  const command = `tar -zcf ../${path.basename(backupPath)}.tar.gz.tmp storage info.json db.json`
-  console.log("EXEC TAR COMMAND:", command)
-  await exec(command, { cwd: backupPath })
+    await fs.promises.rename(tarTmpPath, path.join(parentDir, `${base}.tar.gz`))
+    await log('archive renamed to .tar.gz')
 
-  await fs.promises.rename(backupPath + '.tar.gz.tmp', backupPath + '.tar.gz')
+    await fse.remove(resolvedPath)
+    await log('temp directory removed')
 
-  await fse.remove(backupPath)
+    await log(`createBackup success backupPath=${resolvedPath}`)
+  } catch (err) {
+    const detail = err && err.stack ? err.stack : String(err)
+    await log(`createBackup error: ${detail}`)
+    try {
+      if(await fse.pathExists(resolvedPath)) {
+        await fse.remove(resolvedPath)
+        await log(`cleanup removed temp directory ${resolvedPath}`)
+      }
+    } catch (cleanupErr) {
+      await log(`cleanup temp directory failed: ${cleanupErr}`)
+    }
+    try {
+      if(await fse.pathExists(tarTmpPath)) {
+        await fse.remove(tarTmpPath)
+        await log(`cleanup removed ${tarTmpPath}`)
+      }
+    } catch (cleanupErr) {
+      await log(`cleanup tar tmp failed: ${cleanupErr}`)
+    }
+    throw err
+  }
 }
 
-async function removeOldBackups(backupsDir = '../../backups/', maxAge = 10*24*3600*1000, minBackups = 10) {
-  const backupFiles = await fs.promises.readdir(backupsDir)
+async function removeOldBackups(backupsDir = '../../backups/', maxAge = 10*24*3600*1000, minBackups = 10, log) {
+  const resolvedDir = path.resolve(backupsDir)
+  const dirents = await fs.promises.readdir(resolvedDir, { withFileTypes: true })
   const now = Date.now()
-  // read backup times from filenames
-  const backups = backupFiles.map(file => {
-    const match = file.match(/(\d{4})_(\d{2})_(\d{2})_(\d{2})_(\d{2})_(\d{2})_(\d+)\.tar\.gz/)
-    if(!match) {
-      console.error("BACKUP FILE NAME DOES NOT MATCH:", file)
-      return null
-    }
-    const [_full, year, month, day, hour, minute, second, millis] = match
-    const date = new Date(`${year}-${month}-${day}T${hour}:${minute}:${second}.${millis}Z`)
-    return { file, date }
-  }).filter(backup => backup !== null)
-  // sort by date, from oldest to newest
+
+  const backups = []
+  for(const dirent of dirents) {
+    if(!dirent.isFile()) continue
+    const file = dirent.name
+    if(!file.endsWith('.tar.gz')) continue
+    const date = parseBackupTimestamp(file.replace(/\.tar\.gz$/, ''))
+    if(!date || Number.isNaN(date.getTime())) continue
+    backups.push({ file, date })
+  }
+
   backups.sort((a, b) => a.date - b.date)
   const olderBackups = backups
-      .slice(0, backups.length - minBackups) // leave at least minBackups
-      .filter(backup => now - backup.date > maxAge) // leave only backups older than maxAge
+      .slice(0, backups.length - minBackups)
+      .filter(backup => now - backup.date > maxAge)
   for(const backup of olderBackups) {
-    const backupPath = path.resolve(backupsDir, backup.file)
+    const backupPath = path.join(resolvedDir, backup.file)
     console.log("REMOVING OLD BACKUP:", backupPath)
+    if(log) await log(`removeOldBackups removing archive ${backupPath}`)
     await fse.remove(backupPath)
   }
+
+  for(const dirent of dirents) {
+    if(!dirent.isDirectory()) continue
+    const name = dirent.name
+    const date = parseBackupTimestamp(name)
+    if(!date || Number.isNaN(date.getTime())) continue
+    if(now - date <= maxAge) continue
+    const dirPath = path.join(resolvedDir, name)
+    console.log("REMOVING OLD BACKUP DIRECTORY:", dirPath)
+    if(log) await log(`removeOldBackups removing stale directory ${dirPath}`)
+    await fse.remove(dirPath)
+  }
 }
 
-export { createBackup, currentBackupPath, removeOldBackups }
+export { createBackup, createBackupLogger, currentBackupPath, removeOldBackups }

package/index.js

@@ -8,7 +8,7 @@ import express from 'express'
 import basicAuth from 'express-basic-auth'
 const expressApp = express()
 import fs from 'fs'
-import { createBackup, currentBackupPath, removeOldBackups } from './backup.js'
+import { createBackup, createBackupLogger, currentBackupPath, removeOldBackups } from './backup.js'
 import { restoreBackup } from './restore.js'
 import {
   getLastEventId, removeOldEvents, removeOldOpLogs,
@@ -30,6 +30,8 @@ const backupsDir = config.dir || './backups'
 
 fs.mkdirSync(backupsDir, { recursive: true })
 
+const backupLog = createBackupLogger(backupsDir)
+
 let queue
 (async () => {
   const PQueue = (await import('p-queue')).default
@@ -62,8 +64,8 @@ function doBackup() {
       let lastTriggerTimestamp = await getLastTriggerTimestamp()
       let lastCommandTimestamp = await getLastCommandTimestamp()
 
-      await removeOldBackups(backupsDir, 10 * TWENTY_FOUR_HOURS, 10)
-      await createBackup(path)
+      await removeOldBackups(backupsDir, 10 * TWENTY_FOUR_HOURS, 10, backupLog)
+      await createBackup(path, backupLog)
 
       console.log("====== CLEAR STATS:")
       console.log("Last event id:", lastEventId)
@@ -193,7 +195,7 @@ definition.afterStart(() => {
       console.log('Service init backup:')
       await queue.add(() => doBackup())
       console.log('Service init backup completed')
-    }, TEN_MINUTES)
+    }, 1*60*1000) //TEN_MINUTES)
 
     setInterval(async () => {
       console.log('Daily backup:')

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@live-change/backup-service",
-  "version": "0.9.207",
+  "version": "0.9.209",
   "description": "",
   "main": "index.js",
   "scripts": {
@@ -9,14 +9,14 @@
   "author": "Michał Łaszczewski <michal@emikse.com>",
   "license": "BSD-3-Clause",
   "dependencies": {
-    "@live-change/db-client": "^0.9.207",
-    "@live-change/framework": "^0.9.207",
+    "@live-change/db-client": "^0.9.209",
+    "@live-change/framework": "^0.9.209",
     "express": "^4.18.2",
     "express-basic-auth": "^1.2.1",
     "fs-extra": "^11.1.1",
     "p-queue": "8.1.0",
     "progress-stream": "^2.0.0"
   },
-  "gitHead": "1937f8f9798d5df011b38341119a25afe0def6d1",
+  "gitHead": "e1f934934d5d8a119f7ed27aa97bc6b9e9791719",
   "type": "module"
 }