@live-change/backup-service 0.9.208 → 0.9.212

package/backup.js

@@ -9,6 +9,7 @@ import { exec as cpExec } from 'child_process'
 const exec = util.promisify(cpExec)
 import dump from '@live-change/db-client/lib/dump.js'
 import { once } from 'events'
+import { finished } from 'node:stream/promises'
 import os from "os"
 import path from 'path'
 
@@ -17,6 +18,14 @@ function currentBackupPath(backupsDir = './backups/') {
   return `${backupsDir}/${dateString.substring(0, dateString.length - 1)}`
 }
 
+function createBackupLogger(backupsDir) {
+  const logPath = path.join(path.resolve(backupsDir), 'backups.log')
+  return async function log(message) {
+    const line = `${new Date().toISOString()} ${message}\n`
+    await fs.promises.appendFile(logPath, line, { encoding: 'utf8' })
+  }
+}
+
 async function writeDbBackup(stream) {
   let drainPromise
   async function write(object) {
@@ -27,11 +36,10 @@ async function writeDbBackup(stream) {
       drainPromise = null
     }
   }
-  await dump({
+  try {
+    await dump({
       dao: app.dao,
       db: app.databaseName,
-      //serverUrl: process.env.DB_URL || 'http://localhost:9417/api/ws',
-      //db: process.env.DB_NAME || 'test',
       structure: true,
       verbose: true,
       bucket: 128, // less database stress
@@ -39,64 +47,128 @@ async function writeDbBackup(stream) {
     },
     (method, ...args) => write({ type: 'request', method, parameters: args }),
       () => write({ type: 'sync' })
-  )
-  stream.end()
+    )
+    stream.end()
+    await finished(stream)
+  } catch (err) {
+    stream.destroy()
+    throw err
+  }
 }
 
-async function createBackup(backupPath = currentBackupPath()) {
-  await fs.promises.mkdir(backupPath)
-
-  const dbStream = fs.createWriteStream(path.resolve(backupPath, 'db.json'))
-  await writeDbBackup(dbStream)
-
-  const version = await (
-    fs.promises.readFile('./package.json', { encoding: 'utf8' })
-      .catch(e => 'unknown')
-      .then(data => JSON.parse(data).version)
-  )
-  const info = {
-    version: version,
-    hostname: os.hostname(),
-    directory: path.resolve('.')
-  }
+function parseBackupTimestamp(basename) {
+  const match = basename.match(/^(\d{4})_(\d{2})_(\d{2})_(\d{2})_(\d{2})_(\d{2})_(\d+)/)
+  if(!match) return null
+  const [_full, year, month, day, hour, minute, second, millis] = match
+  return new Date(`${year}-${month}-${day}T${hour}:${minute}:${second}.${millis}Z`)
+}
+
+async function createBackup(backupPath, log) {
+  const resolvedPath = path.resolve(backupPath)
+  const parentDir = path.dirname(resolvedPath)
+  const base = path.basename(resolvedPath)
+  const tarTmpPath = path.join(parentDir, `${base}.tar.gz.tmp`)
+
+  await log(`createBackup start backupPath=${resolvedPath}`)
+
+  try {
+    await fs.promises.mkdir(resolvedPath)
+    await log('mkdir done')
+
+    const dbStream = fs.createWriteStream(path.resolve(resolvedPath, 'db.json'))
+    await writeDbBackup(dbStream)
+    await log('db dump finished (stream closed)')
+
+    const version = await (
+      fs.promises.readFile('./package.json', { encoding: 'utf8' })
+        .catch(e => 'unknown')
+        .then(data => JSON.parse(data).version)
+    )
+    const info = {
+      version: version,
+      hostname: os.hostname(),
+      directory: path.resolve('.')
+    }
+
+    await fs.promises.writeFile(path.resolve(resolvedPath, 'info.json'), JSON.stringify(info, null, '  '))
+    await log('info.json written')
 
-  await fs.promises.writeFile(path.resolve(backupPath, 'info.json'), JSON.stringify(info, null, '  '))
+    await fse.copy("./storage", path.resolve(resolvedPath, "storage"))
+    await log('storage copied')
 
-  await fse.copy("./storage", path.resolve(backupPath, "storage"))
+    const command = `tar -zcf ../${base}.tar.gz.tmp storage info.json db.json`
+    console.log("EXEC TAR COMMAND:", command)
+    await log(`tar start command=${command}`)
+    await exec(command, { cwd: resolvedPath })
+    await log('tar done')
 
-  const command = `tar -zcf ../${path.basename(backupPath)}.tar.gz.tmp storage info.json db.json`
-  console.log("EXEC TAR COMMAND:", command)
-  await exec(command, { cwd: backupPath })
+    await fs.promises.rename(tarTmpPath, path.join(parentDir, `${base}.tar.gz`))
+    await log('archive renamed to .tar.gz')
 
-  await fs.promises.rename(backupPath + '.tar.gz.tmp', backupPath + '.tar.gz')
+    await fse.remove(resolvedPath)
+    await log('temp directory removed')
 
-  await fse.remove(backupPath)
+    await log(`createBackup success backupPath=${resolvedPath}`)
+  } catch (err) {
+    const detail = err && err.stack ? err.stack : String(err)
+    await log(`createBackup error: ${detail}`)
+    try {
+      if(await fse.pathExists(resolvedPath)) {
+        await fse.remove(resolvedPath)
+        await log(`cleanup removed temp directory ${resolvedPath}`)
+      }
+    } catch (cleanupErr) {
+      await log(`cleanup temp directory failed: ${cleanupErr}`)
+    }
+    try {
+      if(await fse.pathExists(tarTmpPath)) {
+        await fse.remove(tarTmpPath)
+        await log(`cleanup removed ${tarTmpPath}`)
+      }
+    } catch (cleanupErr) {
+      await log(`cleanup tar tmp failed: ${cleanupErr}`)
+    }
+    throw err
+  }
 }
 
-async function removeOldBackups(backupsDir = '../../backups/', maxAge = 10*24*3600*1000, minBackups = 10) {
-  const backupFiles = await fs.promises.readdir(backupsDir)
+async function removeOldBackups(backupsDir = '../../backups/', maxAge = 10*24*3600*1000, minBackups = 10, log) {
+  const resolvedDir = path.resolve(backupsDir)
+  const dirents = await fs.promises.readdir(resolvedDir, { withFileTypes: true })
   const now = Date.now()
-  // read backup times from filenames
-  const backups = backupFiles.map(file => {
-    const match = file.match(/(\d{4})_(\d{2})_(\d{2})_(\d{2})_(\d{2})_(\d{2})_(\d+)\.tar\.gz/)
-    if(!match) {
-      console.error("BACKUP FILE NAME DOES NOT MATCH:", file)
-      return null
-    }
-    const [_full, year, month, day, hour, minute, second, millis] = match
-    const date = new Date(`${year}-${month}-${day}T${hour}:${minute}:${second}.${millis}Z`)
-    return { file, date }
-  }).filter(backup => backup !== null)
-  // sort by date, from oldest to newest
+
+  const backups = []
+  for(const dirent of dirents) {
+    if(!dirent.isFile()) continue
+    const file = dirent.name
+    if(!file.endsWith('.tar.gz')) continue
+    const date = parseBackupTimestamp(file.replace(/\.tar\.gz$/, ''))
+    if(!date || Number.isNaN(date.getTime())) continue
+    backups.push({ file, date })
+  }
+
   backups.sort((a, b) => a.date - b.date)
   const olderBackups = backups
-      .slice(0, backups.length - minBackups) // leave at least minBackups
-      .filter(backup => now - backup.date > maxAge) // leave only backups older than maxAge
+      .slice(0, backups.length - minBackups)
+      .filter(backup => now - backup.date > maxAge)
   for(const backup of olderBackups) {
-    const backupPath = path.resolve(backupsDir, backup.file)
+    const backupPath = path.join(resolvedDir, backup.file)
     console.log("REMOVING OLD BACKUP:", backupPath)
+    if(log) await log(`removeOldBackups removing archive ${backupPath}`)
     await fse.remove(backupPath)
   }
+
+  for(const dirent of dirents) {
+    if(!dirent.isDirectory()) continue
+    const name = dirent.name
+    const date = parseBackupTimestamp(name)
+    if(!date || Number.isNaN(date.getTime())) continue
+    if(now - date <= maxAge) continue
+    const dirPath = path.join(resolvedDir, name)
+    console.log("REMOVING OLD BACKUP DIRECTORY:", dirPath)
+    if(log) await log(`removeOldBackups removing stale directory ${dirPath}`)
+    await fse.remove(dirPath)
+  }
 }
 
-export { createBackup, currentBackupPath, removeOldBackups }
+export { createBackup, createBackupLogger, currentBackupPath, removeOldBackups }

package/clear.js

@@ -97,4 +97,35 @@ export async function removeOldCommands(before, options) {
     if(options.delay) await new Promise(resolve => setTimeout(resolve, options.delay))
     more = deleteStats.count >= bucket
   }
+}
+
+const clearEventReportsQuery = `${async (input, output, { indexName, idPrefix, bucket, before }) => {
+  const indexReader = await input.index(indexName)
+  const tableWriter = await output.table('eventReports')
+  const pointers = await indexReader.rangeGet({ limit: bucket, lt: before })
+  for(const pointer of pointers) {
+    await tableWriter.delete(idPrefix + pointer.to)
+  }
+  await output.put({ count: pointers.length })
+}}`
+
+async function removeOldEventReportsForIndex(indexName, idPrefix, before, options) {
+  const bucket = options?.bucket || 128
+  let more = true
+  while(more) {
+    const queryResult = await app.dao.request(['database', 'query'], app.databaseName, clearEventReportsQuery,
+      { indexName, idPrefix, bucket, before })
+    const deleteStats = queryResult[0]
+    if(options?.delay) await new Promise(resolve => setTimeout(resolve, options.delay))
+    more = deleteStats.count >= bucket
+  }
+}
+
+export async function removeOldEventReports(commandBefore, triggerBefore, options = {}) {
+  if(!app.splitCommands && commandBefore) {
+    await removeOldEventReportsForIndex('commands_byTimestamp', 'command_', commandBefore, options)
+  }
+  if(!app.splitTriggers && triggerBefore) {
+    await removeOldEventReportsForIndex('triggers_byTimestamp', 'trigger_', triggerBefore, options)
+  }
 }

package/index.js

@@ -8,17 +8,19 @@ import express from 'express'
 import basicAuth from 'express-basic-auth'
 const expressApp = express()
 import fs from 'fs'
-import { createBackup, currentBackupPath, removeOldBackups } from './backup.js'
+import { createBackup, createBackupLogger, currentBackupPath, removeOldBackups } from './backup.js'
 import { restoreBackup } from './restore.js'
 import {
   getLastEventId, removeOldEvents, removeOldOpLogs,
-  getLastCommandTimestamp, getLastTriggerTimestamp, removeOldTriggers, removeOldCommands
+  getLastCommandTimestamp, getLastTriggerTimestamp, removeOldTriggers, removeOldCommands,
+  removeOldEventReports
 } from './clear.js'
 
 import progress from "progress-stream"
 
 const TWENTY_FOUR_HOURS = 24 * 3600 * 1000
 const TEN_MINUTES = 10 * 60 * 1000
+const DEFAULT_OP_LOG_RETENTION_MS = 60 * 60 * 1000
 
 let currentBackup = null
 
@@ -30,6 +32,8 @@ const backupsDir = config.dir || './backups'
 
 fs.mkdirSync(backupsDir, { recursive: true })
 
+const backupLog = createBackupLogger(backupsDir)
+
 let queue
 (async () => {
   const PQueue = (await import('p-queue')).default
@@ -41,7 +45,7 @@ if(backupServerUsername && backupServerPassword ) {
   users[backupServerUsername] = backupServerPassword
 }
 
-if(Object.keys(users) > 0) {
+if(Object.keys(users).length > 0) {
   expressApp.use(basicAuth({
     users,
     challenge: true,
@@ -49,6 +53,87 @@ if(Object.keys(users) > 0) {
   }))
 }
 
+function resolveRetentionMs(option) {
+  const value = config[option]
+  if(Number.isInteger(value)) return value
+  if(value === true && Number.isInteger(config.retentionMs)) return config.retentionMs
+  if(value !== false && Number.isInteger(config.retentionMs)) return config.retentionMs
+  return null
+}
+
+function shouldClear(option) {
+  const value = config[option]
+  if(value === false) return false
+  if(Number.isInteger(value)) return true
+  if(value === true) return true
+  if(Number.isInteger(config.retentionMs)) return true
+  return false
+}
+
+function resolveOpLogRetentionMs() {
+  if(Number.isInteger(config.opLogRetentionMs)) return config.opLogRetentionMs
+  if(config.clearOpLogs === true && Number.isInteger(config.retentionMs)) return config.retentionMs
+  if(Number.isInteger(config.clearOpLogs)) return config.clearOpLogs
+  return DEFAULT_OP_LOG_RETENTION_MS
+}
+
+function resolveEventReportsRetentionMs() {
+  if(Number.isInteger(config.clearEventReports)) return config.clearEventReports
+  return resolveRetentionMs('clearEventReports')
+    ?? resolveRetentionMs('clearCommands')
+    ?? resolveRetentionMs('clearTriggers')
+}
+
+function computeTimestampCutoff(lastTimestamp, retentionMs) {
+  if(retentionMs === null) return null
+  const date = new Date(lastTimestamp)
+  if(!Number.isInteger(date.getTime())) return null
+  return new Date(date.getTime() - retentionMs).toISOString()
+}
+
+function computeCutoffs({ lastEventId, lastTriggerTimestamp, lastCommandTimestamp }) {
+  const eventsRetention = resolveRetentionMs('clearEvents')
+  let eventsBefore = lastEventId
+  if(shouldClear('clearEvents') && eventsRetention !== null && lastEventId) {
+    const ts = (+(lastEventId.split(':')[0]) - eventsRetention)
+    eventsBefore = ts.toFixed(0).padStart(16, '0') + ':'
+  } else if(!shouldClear('clearEvents')) {
+    eventsBefore = null
+  }
+
+  const triggersRetention = resolveRetentionMs('clearTriggers')
+  let triggersBefore = lastTriggerTimestamp
+  if(shouldClear('clearTriggers')) {
+    triggersBefore = computeTimestampCutoff(lastTriggerTimestamp, triggersRetention)
+  } else {
+    triggersBefore = null
+  }
+
+  const commandsRetention = resolveRetentionMs('clearCommands')
+  let commandsBefore = lastCommandTimestamp
+  if(shouldClear('clearCommands')) {
+    commandsBefore = computeTimestampCutoff(lastCommandTimestamp, commandsRetention)
+  } else {
+    commandsBefore = null
+  }
+
+  const eventReportsRetention = resolveEventReportsRetentionMs()
+  let eventReportsCommandsBefore = null
+  let eventReportsTriggersBefore = null
+  if(shouldClear('clearEventReports') && eventReportsRetention !== null) {
+    eventReportsCommandsBefore = computeTimestampCutoff(lastCommandTimestamp, eventReportsRetention)
+    eventReportsTriggersBefore = computeTimestampCutoff(lastTriggerTimestamp, eventReportsRetention)
+  }
+
+  return {
+    eventsBefore,
+    triggersBefore,
+    commandsBefore,
+    eventReportsCommandsBefore,
+    eventReportsTriggersBefore,
+    opLogRetentionMs: shouldClear('clearOpLogs') ? resolveOpLogRetentionMs() : null
+  }
+}
 
 function doBackup() {
   console.log("DOING BACKUP!")
@@ -62,46 +147,63 @@ function doBackup() {
       let lastTriggerTimestamp = await getLastTriggerTimestamp()
       let lastCommandTimestamp = await getLastCommandTimestamp()
 
-      await removeOldBackups(backupsDir, 10 * TWENTY_FOUR_HOURS, 10)
-      await createBackup(path)
+      await removeOldBackups(backupsDir, 10 * TWENTY_FOUR_HOURS, 10, backupLog)
+      await createBackup(path, backupLog)
 
       console.log("====== CLEAR STATS:")
       console.log("Last event id:", lastEventId)
       console.log("Last trigger timestamp:", lastTriggerTimestamp)
       console.log("Last command timestamp:", lastCommandTimestamp)
 
-      if(Number.isInteger(config.clearEvents) && lastEventId) {
-        const ts = (+(lastEventId.split(':')[0]) - config.clearEvents)
-        console.log("Clear events before", new Date(ts))
-        lastEventId = ts.toFixed(0).padStart(16, '0')+':'
-      }
-      if(Number.isInteger(config.clearTriggers) && Number.isInteger(new Date(lastTriggerTimestamp).getTime())) {
-        const date = new Date(lastTriggerTimestamp)
-        const old = new Date(date.getTime() - config.clearTriggers)
-        console.log("Clear triggers before", old)
-        lastTriggerTimestamp = old.toISOString()
+      const cutoffs = computeCutoffs({ lastEventId, lastTriggerTimestamp, lastCommandTimestamp })
+
+      if(cutoffs.eventsBefore) console.log("Clear events before", new Date(+(cutoffs.eventsBefore.split(':')[0])))
+      if(cutoffs.triggersBefore) console.log("Clear triggers before", cutoffs.triggersBefore)
+      if(cutoffs.commandsBefore) console.log("Clear commands before", cutoffs.commandsBefore)
+      if(cutoffs.eventReportsCommandsBefore) {
+        console.log("Clear eventReports (commands) before", cutoffs.eventReportsCommandsBefore)
       }
-      if(Number.isInteger(config.clearCommands) && Number.isInteger(new Date(lastCommandTimestamp).getTime())) {
-        const date = new Date(lastCommandTimestamp)
-        const old = new Date(date.getTime() - config.clearCommands)
-        console.log("Clear commands before", old)
-        lastCommandTimestamp = old.toISOString()
+      if(cutoffs.eventReportsTriggersBefore) {
+        console.log("Clear eventReports (triggers) before", cutoffs.eventReportsTriggersBefore)
       }
 
       console.log("====== CLEARING:")
-      console.log("Last remaining event id:", lastEventId)
-      console.log("Last remaining trigger timestamp:", lastTriggerTimestamp)
-      console.log("Last remaining command timestamp:", lastCommandTimestamp)
-
-      if(config.clearCommands) await removeOldCommands(lastCommandTimestamp, { delay: 10 })
-      if(config.clearTriggers) await removeOldTriggers(lastTriggerTimestamp, { delay: 10 })
-      if(config.clearEvents) await removeOldEvents(lastEventId)
-      if(config.clearOpLogs) await removeOldOpLogs()
+      console.log("Last remaining event id:", cutoffs.eventsBefore)
+      console.log("Last remaining trigger timestamp:", cutoffs.triggersBefore)
+      console.log("Last remaining command timestamp:", cutoffs.commandsBefore)
+
+      if(shouldClear('clearEventReports')) {
+        await backupLog('clearEventReports start')
+        await removeOldEventReports(
+          cutoffs.eventReportsCommandsBefore,
+          cutoffs.eventReportsTriggersBefore,
+          { delay: 10 }
+        )
+        await backupLog('clearEventReports done')
+      }
+
+      if(shouldClear('clearCommands') && cutoffs.commandsBefore) {
+        await removeOldCommands(cutoffs.commandsBefore, { delay: 10 })
+      }
+      if(shouldClear('clearTriggers') && cutoffs.triggersBefore) {
+        await removeOldTriggers(cutoffs.triggersBefore, { delay: 10 })
+      }
+      if(shouldClear('clearEvents') && cutoffs.eventsBefore) {
+        await removeOldEvents(cutoffs.eventsBefore)
+      }
+      if(shouldClear('clearOpLogs') && cutoffs.opLogRetentionMs !== null) {
+        await removeOldOpLogs(cutoffs.opLogRetentionMs)
+      }
     })
   }
-  currentBackup.promise.then(done => {
+  currentBackup.promise.then(async () => {
     console.log("BACKUP CREATED!")
-    fs.promises.writeFile(backupsDir + '/latest.txt', filename+'.tar.gz')
+    await fs.promises.writeFile(backupsDir + '/latest.txt', filename+'.tar.gz')
+    currentBackup = null
+  }).catch(async err => {
+    const detail = err && err.stack ? err.stack : String(err)
+    console.error("BACKUP FAILED:", err)
+    await backupLog(`backup failed: ${detail}`)
     currentBackup = null
   })
   return currentBackup
@@ -112,7 +214,7 @@ expressApp.get('/backup/doBackup', async (req, res) => {
     res.status(200).send('Backup in progress: ' + currentBackup.filename)
     return
   }
-  await doBackup()
+  await queue.add(() => doBackup())
   res.status(200).send('Creating backup: '+currentBackup.filename)
 })
 
@@ -202,4 +304,4 @@ definition.afterStart(() => {
     }, TWENTY_FOUR_HOURS)
   })
 
-})
+})

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@live-change/backup-service",
-  "version": "0.9.208",
+  "version": "0.9.212",
   "description": "",
   "main": "index.js",
   "scripts": {
@@ -9,14 +9,14 @@
   "author": "Michał Łaszczewski <michal@emikse.com>",
   "license": "BSD-3-Clause",
   "dependencies": {
-    "@live-change/db-client": "^0.9.208",
-    "@live-change/framework": "^0.9.208",
+    "@live-change/db-client": "^0.9.212",
+    "@live-change/framework": "^0.9.212",
     "express": "^4.18.2",
     "express-basic-auth": "^1.2.1",
     "fs-extra": "^11.1.1",
-    "p-queue": "8.1.0",
+    "p-queue": "^8.1.1",
     "progress-stream": "^2.0.0"
   },
-  "gitHead": "8818bbf0c3db428fec1f578f615e00232d3f895f",
+  "gitHead": "40268236c57d5f123c1e8f1815c310464d043d52",
   "type": "module"
 }