@live-change/backup-service 0.8.3

package/backup.js

@@ -0,0 +1,98 @@
+import App from '@live-change/framework'
+const app = App.app()
+import fs from 'fs'
+import fse from 'fs-extra'
+import util from 'util'
+import { exec as cpExec } from 'child_process'
+const exec = util.promisify(cpExec)
+import dump from '@live-change/db-client/lib/dump.js'
+import { once } from 'events'
+import os from "os"
+import path from 'path'
+
+function currentBackupPath(backupsDir = './backups/') {
+  const dateString = new Date().toISOString().slice(0,-1).replace(/[T:\\.-]/gi, '_')
+  return `${backupsDir}/${dateString.substring(0, dateString.length - 1)}`
+}
+
+async function writeDbBackup(stream) {
+  let drainPromise
+  async function write(object) {
+    const code = JSON.stringify(object)
+    if(!stream.write(code + '\n')) {
+      if(!drainPromise) drainPromise = once(stream, 'drain')
+      await drainPromise
+      drainPromise = null
+    }
+  }
+  await dump({
+      dao: app.dao,
+      db: app.databaseName,
+      //serverUrl: process.env.DB_URL || 'http://localhost:9417/api/ws',
+      //db: process.env.DB_NAME || 'test',
+      structure: true,
+      verbose: true
+    },
+    (method, ...args) => write({ type: 'request', method, parameters: args }),
+      () => write({ type: 'sync' })
+  )
+  stream.end()
+}
+
+async function createBackup(backupPath = currentBackupPath()) {
+  await fs.promises.mkdir(backupPath)
+
+  const dbStream = fs.createWriteStream(path.resolve(backupPath, 'db.json'))
+  await writeDbBackup(dbStream)
+
+  const version = await (
+    fs.promises.readFile('./package.json', { encoding: 'utf8' })
+      .catch(e => 'unknown')
+      .then(data => JSON.parse(data).version)
+  )
+  const info = {
+    version: version,
+    hostname: os.hostname(),
+    directory: path.resolve('.')
+  }
+
+  await fs.promises.writeFile(path.resolve(backupPath, 'info.json'), JSON.stringify(info, null, '  '))
+
+  await fse.copy("./storage", path.resolve(backupPath, "storage"))
+
+  const command = `tar -zcf ../${path.basename(backupPath)}.tar.gz.tmp storage info.json db.json`
+  console.log("EXEC TAR COMMAND:", command)
+  await exec(command, { cwd: backupPath })
+
+  await fs.promises.rename(backupPath + '.tar.gz.tmp', backupPath + '.tar.gz')
+
+  await fse.remove(backupPath)
+}
+
+async function removeOldBackups(backupsDir = '../../backups/', maxAge = 10*24*3600*1000, minBackups = 10) {
+  const backupFiles = await fs.promises.readdir(backupsDir)
+  const now = Date.now()
+  // read backup times from filenames
+  const backups = backupFiles.map(file => {
+    const match = file.match(/(\d{4})_(\d{2})_(\d{2})_(\d{2})_(\d{2})_(\d{2})_(\d+)\.tar\.gz/)
+    if(!match) {
+      console.error("BACKUP FILE NAME DOES NOT MATCH:", file)
+      return null
+    }
+    const [_full, year, month, day, hour, minute, second, millis] = match
+    const date = new Date(`${year}-${month}-${day}T${hour}:${minute}:${second}.${millis}Z`)
+    return { file, date }
+  }).filter(backup => backup !== null)
+  // sort by date, from oldest to newest
+  backups.sort((a, b) => a.date - b.date)
+  const olderBackups = backups
+      .slice(0, backups.length - minBackups) // leave at least minBackups
+      .filter(backup => now - backup.date > maxAge) // leave only backups older than maxAge
+  for(const backup of olderBackups) {
+    const backupPath = path.resolve(backupsDir, backup.file)
+    console.log("REMOVING OLD BACKUP:", backupPath)
+    await fse.remove(backupPath)
+  }
+}
+
+export { createBackup, currentBackupPath, removeOldBackups }

package/definition.js

@@ -0,0 +1,11 @@
+import App from '@live-change/framework'
+const app = App.app()
+
+import relationsPlugin from '@live-change/relations-plugin'
+
+const definition = app.createServiceDefinition({
+  name: "backup",
+  use: [ relationsPlugin ]
+})
+
+export default definition

package/download_backup.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+BACKUP_SERVER_URL=$1
+BACKUP_SERVER_USERNAME=$2
+BACKUP_SERVER_PASSWORD=$3
+
+mkdir -p backups
+
+echo Checking backup on "$BACKUP_SERVER_URL/latest"
+
+LATEST_BACKUP_URL=`curl -L -u "$BACKUP_SERVER_USERNAME:$BACKUP_SERVER_PASSWORD" "$BACKUP_SERVER_URL/latest"`
+echo Downloading latest backup from url $LATEST_BACKUP_URL
+pushd backups
+  curl -L -u "$BACKUP_SERVER_USERNAME:$BACKUP_SERVER_PASSWORD" $LATEST_BACKUP_URL -O -J
+  echo "$(basename -- $LATEST_BACKUP_URL)" > latest.txt
+popd

package/index.js

@@ -0,0 +1,166 @@
+import App from '@live-change/framework'
+const app = App.app()
+import definition from './definition.js'
+const config = definition.config
+export default definition
+
+import express from 'express'
+import basicAuth from 'express-basic-auth'
+const expressApp = express()
+import fs from 'fs'
+import { createBackup, currentBackupPath, removeOldBackups } from './backup.js'
+import { restoreBackup } from './restore.js'
+
+import progress from "progress-stream"
+
+const TWENTY_FOUR_HOURS = 24 * 3600 * 1000
+const TEN_MINUTES = 10 * 60 * 1000
+
+let currentBackup = null
+
+const backupServerPort = config.port || process.env.BACKUP_SERVER_PORT || 8007
+const backupServerUsername = config.username || process.env.BACKUP_SERVER_USERNAME
+const backupServerPassword = config.password || process.env.BACKUP_SERVER_PASSWORD
+
+const backupsDir = config.dir || './backups'
+
+fs.mkdirSync(backupsDir, { recursive: true })
+
+let queue
+(async () => {
+  const PQueue = (await import('p-queue')).default
+  queue = new PQueue({ concurrency: 1 })
+})()
+
+let users = config.users || {}
+if(backupServerUsername && backupServerPassword ) {
+  users[backupServerUsername] = backupServerPassword
+}
+
+if(Object.keys(users) > 0) {
+  expressApp.use(basicAuth({
+    users,
+    challenge: true,
+    realm: 'backupServer',
+  }))
+}
+
+function doBackup() {
+  if(currentBackup) return currentBackup
+  const path = currentBackupPath(backupsDir)
+  const filename = path.slice(path.lastIndexOf('/')+1)
+  currentBackup = {
+    path, filename,
+    promise: queue.add(async () => {
+      await removeOldBackups(backupsDir, 10*TWENTY_FOUR_HOURS, 10)
+      await createBackup(path)
+    })
+  }
+  currentBackup.promise.then(done => {
+    console.log("BACKUP CREATED!")
+    fs.promises.writeFile(backupsDir + '/latest.txt', filename+'.tar.gz')
+    currentBackup = null
+  })
+  return currentBackup
+}
+
+expressApp.get('/backup/doBackup', async (req, res) => {
+  if(currentBackup) {
+    res.status(200).send('Backup in progress: ' + currentBackup.filename)
+    return
+  }
+  await doBackup()
+  res.status(200).send('Creating backup: '+currentBackup.filename)
+})
+
+expressApp.get('/backup/latest', async (req, res) => {
+  const latest = await fs.promises.readFile(backupsDir + '/latest.txt', { encoding: 'utf8' })
+  res.status(200).send(`https://${req.get('host')}/backup/download/${latest}`)
+})
+
+expressApp.get('/backup', async (req, res) => {
+  try {
+    const files = await fs.promises.readdir(backupsDir)
+    let backupFiles = files.filter(file => file.endsWith('tar.gz'))
+        .map(fileName => `https://${req.get('host')}/backup/download/${fileName}`)
+        .reverse()
+        .join('\n')
+
+    res.status(200).header('Content-Type', 'text/plain').send(backupFiles)
+  } catch(err) {
+    console.log(err)
+  }
+})
+
+expressApp.get('/backup/download/:fileName', (req, res) => {
+  const { fileName } = req.params
+  const file = `${backupsDir}/${fileName}`
+  res.status(200).header('Content-Disposition', `attachment; filename="${fileName}"`).download(file)
+})
+
+expressApp.post('/backup/upload/:fileName', async (req, res) => {
+  const { fileName } = req.params
+  const file = `${backupsDir}/${fileName}`
+
+  const size = +req.header('Content-Length')
+  console.log("SIZE", size)
+
+  const uploadProgress = progress({
+    length: size,
+    time: 600
+  })
+  req.pipe(uploadProgress)
+  const fileStream = fs.createWriteStream(file)
+  uploadProgress.pipe(fileStream)
+
+  fileStream.on('finish', () => {
+    res.status(200)
+    res.send("" + uploadProgress.progress().transferred)
+  })
+
+  fileStream.on('error', error => {
+    res.status(503)
+    res.send("Error " + error.toString())
+  })
+})
+
+let restoringBackup = false
+expressApp.post('/restore/:fileName', async (req, res) => {
+  const { fileName } = req.params
+  throw new Error('not implemented')
+  if(restoringBackup) {
+    res.status(500)
+    res.send(`Restoring backup ${restoringBackup} in progress...`)
+    return
+  }
+  restoringBackup = fileName
+  const file = `${backupsDir}/${fileName}`
+  await restoreBackup({
+    file
+  })
+  restoringBackup = null
+  return 'ok'
+})
+
+process.on('unhandledRejection', (reason, p) => {
+  console.log('Unhandled Rejection at: Promise', p, 'reason:', reason)
+})
+
+definition.beforeStart(() => {
+
+  expressApp.listen(backupServerPort, () => {
+    console.log(`backup port listening on ${backupServerPort}`)
+    setTimeout(async () => {
+      console.log('Service init backup:')
+      await queue.add(() => doBackup())
+      console.log('Service init backup completed')
+    }, TEN_MINUTES)
+
+    setInterval(async () => {
+      console.log('Daily backup:')
+      await queue.add(() => doBackup())
+      console.log('Daily backup completed')
+    }, TWENTY_FOUR_HOURS)
+  })
+
+})

package/package.json

@@ -0,0 +1,22 @@
+{
+  "name": "@live-change/backup-service",
+  "version": "0.8.3",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "test": "NODE_ENV=test blue-tape tests/*"
+  },
+  "author": "Michał Łaszczewski <michal@emikse.com>",
+  "license": "BSD-3-Clause",
+  "dependencies": {
+    "@live-change/db-client": "^0.8.3",
+    "@live-change/framework": "^0.8.3",
+    "express": "^4.18.2",
+    "express-basic-auth": "^1.2.1",
+    "fs-extra": "^11.1.1",
+    "p-queue": "^7.3.4",
+    "progress-stream": "^2.0.0"
+  },
+  "gitHead": "81b4ac5a36ab1abdf9d521e3c4fbea6d194bb892",
+  "type": "module"
+}

package/restore.js

@@ -0,0 +1,12 @@
+import fs from 'fs'
+import fse from 'fs-extra'
+import util from 'util'
+import { exec } from "child_process"
+const execProcess = util.promisify(exec)
+import { once } from 'events'
+import os from "os"
+import path from 'path'
+
+export function restoreBackup({ file }) {
+  throw new Error('not_implemented')
+}

package/upload_backup.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+BACKUP_SERVER_URL=$1
+BACKUP_SERVER_USERNAME=$2
+BACKUP_SERVER_PASSWORD=$3
+BACKUP_FILE=$4
+BACKUP_NAME=(basename $BACKUP_FILE)
+
+mkdir -p backups
+
+echo Uploading backup $BACKUP_NAME
+UPLOAD_BACKUP_URL="$BACKUP_SERVER_URL/upload/$BACKUP_NAME"`
+pushd backups
+  curl -L -u --data-binary "@$BACKUP_FILE" "$BACKUP_SERVER_USERNAME:$BACKUP_SERVER_PASSWORD" $UPLOAD_BACKUP_URL
+popd