@live-change/backup-service 0.9.209 → 0.9.212

package/clear.js

@@ -97,4 +97,35 @@ export async function removeOldCommands(before, options) {
     if(options.delay) await new Promise(resolve => setTimeout(resolve, options.delay))
     more = deleteStats.count >= bucket
   }
+}
+
+const clearEventReportsQuery = `${async (input, output, { indexName, idPrefix, bucket, before }) => {
+  const indexReader = await input.index(indexName)
+  const tableWriter = await output.table('eventReports')
+  const pointers = await indexReader.rangeGet({ limit: bucket, lt: before })
+  for(const pointer of pointers) {
+    await tableWriter.delete(idPrefix + pointer.to)
+  }
+  await output.put({ count: pointers.length })
+}}`
+
+async function removeOldEventReportsForIndex(indexName, idPrefix, before, options) {
+  const bucket = options?.bucket || 128
+  let more = true
+  while(more) {
+    const queryResult = await app.dao.request(['database', 'query'], app.databaseName, clearEventReportsQuery,
+      { indexName, idPrefix, bucket, before })
+    const deleteStats = queryResult[0]
+    if(options?.delay) await new Promise(resolve => setTimeout(resolve, options.delay))
+    more = deleteStats.count >= bucket
+  }
+}
+
+export async function removeOldEventReports(commandBefore, triggerBefore, options = {}) {
+  if(!app.splitCommands && commandBefore) {
+    await removeOldEventReportsForIndex('commands_byTimestamp', 'command_', commandBefore, options)
+  }
+  if(!app.splitTriggers && triggerBefore) {
+    await removeOldEventReportsForIndex('triggers_byTimestamp', 'trigger_', triggerBefore, options)
+  }
 }

package/index.js

@@ -12,13 +12,15 @@ import { createBackup, createBackupLogger, currentBackupPath, removeOldBackups }
 import { restoreBackup } from './restore.js'
 import {
   getLastEventId, removeOldEvents, removeOldOpLogs,
-  getLastCommandTimestamp, getLastTriggerTimestamp, removeOldTriggers, removeOldCommands
+  getLastCommandTimestamp, getLastTriggerTimestamp, removeOldTriggers, removeOldCommands,
+  removeOldEventReports
 } from './clear.js'
 
 import progress from "progress-stream"
 
 const TWENTY_FOUR_HOURS = 24 * 3600 * 1000
 const TEN_MINUTES = 10 * 60 * 1000
+const DEFAULT_OP_LOG_RETENTION_MS = 60 * 60 * 1000
 
 let currentBackup = null
 
@@ -43,7 +45,7 @@ if(backupServerUsername && backupServerPassword ) {
   users[backupServerUsername] = backupServerPassword
 }
 
-if(Object.keys(users) > 0) {
+if(Object.keys(users).length > 0) {
   expressApp.use(basicAuth({
     users,
     challenge: true,
@@ -51,6 +53,87 @@ if(Object.keys(users) > 0) {
   }))
 }
 
+function resolveRetentionMs(option) {
+  const value = config[option]
+  if(Number.isInteger(value)) return value
+  if(value === true && Number.isInteger(config.retentionMs)) return config.retentionMs
+  if(value !== false && Number.isInteger(config.retentionMs)) return config.retentionMs
+  return null
+}
+
+function shouldClear(option) {
+  const value = config[option]
+  if(value === false) return false
+  if(Number.isInteger(value)) return true
+  if(value === true) return true
+  if(Number.isInteger(config.retentionMs)) return true
+  return false
+}
+
+function resolveOpLogRetentionMs() {
+  if(Number.isInteger(config.opLogRetentionMs)) return config.opLogRetentionMs
+  if(config.clearOpLogs === true && Number.isInteger(config.retentionMs)) return config.retentionMs
+  if(Number.isInteger(config.clearOpLogs)) return config.clearOpLogs
+  return DEFAULT_OP_LOG_RETENTION_MS
+}
+
+function resolveEventReportsRetentionMs() {
+  if(Number.isInteger(config.clearEventReports)) return config.clearEventReports
+  return resolveRetentionMs('clearEventReports')
+    ?? resolveRetentionMs('clearCommands')
+    ?? resolveRetentionMs('clearTriggers')
+}
+
+function computeTimestampCutoff(lastTimestamp, retentionMs) {
+  if(retentionMs === null) return null
+  const date = new Date(lastTimestamp)
+  if(!Number.isInteger(date.getTime())) return null
+  return new Date(date.getTime() - retentionMs).toISOString()
+}
+
+function computeCutoffs({ lastEventId, lastTriggerTimestamp, lastCommandTimestamp }) {
+  const eventsRetention = resolveRetentionMs('clearEvents')
+  let eventsBefore = lastEventId
+  if(shouldClear('clearEvents') && eventsRetention !== null && lastEventId) {
+    const ts = (+(lastEventId.split(':')[0]) - eventsRetention)
+    eventsBefore = ts.toFixed(0).padStart(16, '0') + ':'
+  } else if(!shouldClear('clearEvents')) {
+    eventsBefore = null
+  }
+
+  const triggersRetention = resolveRetentionMs('clearTriggers')
+  let triggersBefore = lastTriggerTimestamp
+  if(shouldClear('clearTriggers')) {
+    triggersBefore = computeTimestampCutoff(lastTriggerTimestamp, triggersRetention)
+  } else {
+    triggersBefore = null
+  }
+
+  const commandsRetention = resolveRetentionMs('clearCommands')
+  let commandsBefore = lastCommandTimestamp
+  if(shouldClear('clearCommands')) {
+    commandsBefore = computeTimestampCutoff(lastCommandTimestamp, commandsRetention)
+  } else {
+    commandsBefore = null
+  }
+
+  const eventReportsRetention = resolveEventReportsRetentionMs()
+  let eventReportsCommandsBefore = null
+  let eventReportsTriggersBefore = null
+  if(shouldClear('clearEventReports') && eventReportsRetention !== null) {
+    eventReportsCommandsBefore = computeTimestampCutoff(lastCommandTimestamp, eventReportsRetention)
+    eventReportsTriggersBefore = computeTimestampCutoff(lastTriggerTimestamp, eventReportsRetention)
+  }
+
+  return {
+    eventsBefore,
+    triggersBefore,
+    commandsBefore,
+    eventReportsCommandsBefore,
+    eventReportsTriggersBefore,
+    opLogRetentionMs: shouldClear('clearOpLogs') ? resolveOpLogRetentionMs() : null
+  }
+}
 
 function doBackup() {
   console.log("DOING BACKUP!")
@@ -72,38 +155,55 @@ function doBackup() {
       console.log("Last trigger timestamp:", lastTriggerTimestamp)
       console.log("Last command timestamp:", lastCommandTimestamp)
 
-      if(Number.isInteger(config.clearEvents) && lastEventId) {
-        const ts = (+(lastEventId.split(':')[0]) - config.clearEvents)
-        console.log("Clear events before", new Date(ts))
-        lastEventId = ts.toFixed(0).padStart(16, '0')+':'
-      }
-      if(Number.isInteger(config.clearTriggers) && Number.isInteger(new Date(lastTriggerTimestamp).getTime())) {
-        const date = new Date(lastTriggerTimestamp)
-        const old = new Date(date.getTime() - config.clearTriggers)
-        console.log("Clear triggers before", old)
-        lastTriggerTimestamp = old.toISOString()
+      const cutoffs = computeCutoffs({ lastEventId, lastTriggerTimestamp, lastCommandTimestamp })
+
+      if(cutoffs.eventsBefore) console.log("Clear events before", new Date(+(cutoffs.eventsBefore.split(':')[0])))
+      if(cutoffs.triggersBefore) console.log("Clear triggers before", cutoffs.triggersBefore)
+      if(cutoffs.commandsBefore) console.log("Clear commands before", cutoffs.commandsBefore)
+      if(cutoffs.eventReportsCommandsBefore) {
+        console.log("Clear eventReports (commands) before", cutoffs.eventReportsCommandsBefore)
       }
-      if(Number.isInteger(config.clearCommands) && Number.isInteger(new Date(lastCommandTimestamp).getTime())) {
-        const date = new Date(lastCommandTimestamp)
-        const old = new Date(date.getTime() - config.clearCommands)
-        console.log("Clear commands before", old)
-        lastCommandTimestamp = old.toISOString()
+      if(cutoffs.eventReportsTriggersBefore) {
+        console.log("Clear eventReports (triggers) before", cutoffs.eventReportsTriggersBefore)
       }
 
       console.log("====== CLEARING:")
-      console.log("Last remaining event id:", lastEventId)
-      console.log("Last remaining trigger timestamp:", lastTriggerTimestamp)
-      console.log("Last remaining command timestamp:", lastCommandTimestamp)
-
-      if(config.clearCommands) await removeOldCommands(lastCommandTimestamp, { delay: 10 })
-      if(config.clearTriggers) await removeOldTriggers(lastTriggerTimestamp, { delay: 10 })
-      if(config.clearEvents) await removeOldEvents(lastEventId)
-      if(config.clearOpLogs) await removeOldOpLogs()
+      console.log("Last remaining event id:", cutoffs.eventsBefore)
+      console.log("Last remaining trigger timestamp:", cutoffs.triggersBefore)
+      console.log("Last remaining command timestamp:", cutoffs.commandsBefore)
+
+      if(shouldClear('clearEventReports')) {
+        await backupLog('clearEventReports start')
+        await removeOldEventReports(
+          cutoffs.eventReportsCommandsBefore,
+          cutoffs.eventReportsTriggersBefore,
+          { delay: 10 }
+        )
+        await backupLog('clearEventReports done')
+      }
+
+      if(shouldClear('clearCommands') && cutoffs.commandsBefore) {
+        await removeOldCommands(cutoffs.commandsBefore, { delay: 10 })
+      }
+      if(shouldClear('clearTriggers') && cutoffs.triggersBefore) {
+        await removeOldTriggers(cutoffs.triggersBefore, { delay: 10 })
+      }
+      if(shouldClear('clearEvents') && cutoffs.eventsBefore) {
+        await removeOldEvents(cutoffs.eventsBefore)
+      }
+      if(shouldClear('clearOpLogs') && cutoffs.opLogRetentionMs !== null) {
+        await removeOldOpLogs(cutoffs.opLogRetentionMs)
+      }
     })
   }
-  currentBackup.promise.then(done => {
+  currentBackup.promise.then(async () => {
     console.log("BACKUP CREATED!")
-    fs.promises.writeFile(backupsDir + '/latest.txt', filename+'.tar.gz')
+    await fs.promises.writeFile(backupsDir + '/latest.txt', filename+'.tar.gz')
+    currentBackup = null
+  }).catch(async err => {
+    const detail = err && err.stack ? err.stack : String(err)
+    console.error("BACKUP FAILED:", err)
+    await backupLog(`backup failed: ${detail}`)
     currentBackup = null
   })
   return currentBackup
@@ -114,7 +214,7 @@ expressApp.get('/backup/doBackup', async (req, res) => {
     res.status(200).send('Backup in progress: ' + currentBackup.filename)
     return
   }
-  await doBackup()
+  await queue.add(() => doBackup())
   res.status(200).send('Creating backup: '+currentBackup.filename)
 })
 
@@ -195,7 +295,7 @@ definition.afterStart(() => {
       console.log('Service init backup:')
       await queue.add(() => doBackup())
       console.log('Service init backup completed')
-    }, 1*60*1000) //TEN_MINUTES)
+    }, TEN_MINUTES)
 
     setInterval(async () => {
       console.log('Daily backup:')
@@ -204,4 +304,4 @@ definition.afterStart(() => {
     }, TWENTY_FOUR_HOURS)
   })
 
-})
+})

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@live-change/backup-service",
-  "version": "0.9.209",
+  "version": "0.9.212",
   "description": "",
   "main": "index.js",
   "scripts": {
@@ -9,14 +9,14 @@
   "author": "Michał Łaszczewski <michal@emikse.com>",
   "license": "BSD-3-Clause",
   "dependencies": {
-    "@live-change/db-client": "^0.9.209",
-    "@live-change/framework": "^0.9.209",
+    "@live-change/db-client": "^0.9.212",
+    "@live-change/framework": "^0.9.212",
     "express": "^4.18.2",
     "express-basic-auth": "^1.2.1",
     "fs-extra": "^11.1.1",
-    "p-queue": "8.1.0",
+    "p-queue": "^8.1.1",
     "progress-stream": "^2.0.0"
   },
-  "gitHead": "e1f934934d5d8a119f7ed27aa97bc6b9e9791719",
+  "gitHead": "40268236c57d5f123c1e8f1815c310464d043d52",
   "type": "module"
 }