@live-change/backup-service 0.8.137 → 0.8.139

package/backup.js

@@ -33,7 +33,9 @@ async function writeDbBackup(stream) {
       //serverUrl: process.env.DB_URL || 'http://localhost:9417/api/ws',
       //db: process.env.DB_NAME || 'test',
       structure: true,
-      verbose: true
+      verbose: true,
+      bucket: 128, // less database stress
+      delay: 10 // less database stress
     },
     (method, ...args) => write({ type: 'request', method, parameters: args }),
       () => write({ type: 'sync' })

package/clear.js

@@ -0,0 +1,95 @@
+import App from '@live-change/framework'
+const app = App.app()
+import definition from './definition.js'
+
+export async function getLastEventId() {
+  if(app.splitEvents) {
+    throw new Error("getLastEventId not supported in split events mode")
+  } else {
+    const lastLogs = await app.dao.get(['database', 'logRange', app.databaseName, 'events', { reverse: true, limit: 1 }])
+    const lastLog = lastLogs[0]
+    return lastLog?.id || ''
+  }
+}
+
+export async function removeOldEvents(before) {
+  if(app.splitEvents) {
+    throw new Error("removeOldEvents not supported in split events mode")
+  } else {
+    await app.dao.request(['database', 'clearLog'], app.databaseName, 'events', before)
+  }
+}
+
+export async function removeOldOpLogs(age = 60*60*1000) {
+  const clearLimit = 40
+  let cleared = 0
+  do {
+    const clearResults = await app.dao.requestWithSettings({ requestTimeout: 100*1e3 },
+      ['database', 'clearDatabaseOpLogs'], app.databaseName, Date.now() - age, clearLimit)
+    cleared = 0
+    for(const result of clearResults.results) if(result.count > cleared) cleared = result.count
+    //console.log("REMOVED", clearResults.count, "OP LOGS", cleared, "MAX")
+    await new Promise(resolve => setTimeout(resolve, 50)) // reduceDatabaseStress
+  } while(cleared === clearLimit)
+}
+
+export async function getLastTriggerTimestamp() {
+  if(app.splitTriggers) {
+    throw new Error("getLastTrigger not supported in split triggers mode")
+  }
+  const lastTriggers = await app.dao.get(
+    ['database', 'indexRange', app.databaseName, 'triggers_byTimestamp', { reverse: true, limit: 1 }])
+  const lastTrigger = lastTriggers[0]?.id
+  return lastTrigger ?? null
+}
+
+export async function getLastCommandTimestamp() {
+  if(app.splitCommands) {
+    throw new Error("getLastCommand not supported in split commands mode")
+  }
+  const lastCommands = await app.dao.get(
+    ['database', 'indexRange', app.databaseName, 'commands_byTimestamp', { reverse: true, limit: 1 }])
+  const lastCommand = lastCommands[0]?.id
+  return lastCommand ?? null
+}
+
+const clearQuery = `${async (input, output, { tableName, bucket, before }) => {
+  const indexReader = await input.index(tableName + '_byTimestamp')
+  const tableWriter = await output.table(tableName)
+  const range = { limit: bucket, lt: before }
+  const pointers = await indexReader.get(range)
+  for(const pointer of pointers) {
+    await tableWriter.delete(pointer.to)
+  }
+  await output.put({ count: pointers.length })
+}}`
+
+export async function removeOldTriggers(before, options) {
+  const bucket = options?.bucket || 128
+  if(app.splitTriggers) {
+    throw new Error("removeOldTriggers not supported in split triggers mode")
+  }
+  let more = true
+  while(more) {
+    const queryResult = await app.dao.request(['database', 'query'], app.databaseName, clearQuery,
+      { tableName: 'triggers', bucket, before })
+    const deleteStats = queryResult[0]
+    if(options.delay) await new Promise(resolve => setTimeout(resolve, options.delay))
+    more = deleteStats.count >= bucket
+  }
+}
+
+export async function removeOldCommands(before, options) {
+  const bucket = options?.bucket || 128
+  if(app.splitCommands) {
+    throw new Error("removeOldCommands not supported in split commands mode")
+  }
+  let more = true
+  while(more) {
+    const queryResult = await app.dao.request(['database', 'query'], app.databaseName, clearQuery,
+      { tableName: 'commands', bucket, before })
+    const deleteStats = queryResult[0]
+    if(options.delay) await new Promise(resolve => setTimeout(resolve, options.delay))
+    more = deleteStats.count >= bucket
+  }
+}

package/index.js

@@ -10,7 +10,10 @@ const expressApp = express()
 import fs from 'fs'
 import { createBackup, currentBackupPath, removeOldBackups } from './backup.js'
 import { restoreBackup } from './restore.js'
-import { getLastEventId, removeOldEvents } from './clearEvents.js'
+import {
+  getLastEventId, removeOldEvents, removeOldOpLogs,
+  getLastCommandTimestamp, getLastTriggerTimestamp, removeOldTriggers, removeOldCommands
+} from './clear.js'
 
 import progress from "progress-stream"
 
@@ -46,6 +49,7 @@ if(Object.keys(users) > 0) {
   }))
 }
 
+
 function doBackup() {
   console.log("DOING BACKUP!")
   if(currentBackup) return currentBackup
@@ -54,10 +58,45 @@ function doBackup() {
   currentBackup = {
     path, filename,
     promise: queue.add(async () => {
-      const lastEventId = await getLastEventId()
+      let lastEventId = await getLastEventId()
+      let lastTriggerTimestamp = await getLastTriggerTimestamp()
+      let lastCommandTimestamp = await getLastCommandTimestamp()
+
       await removeOldBackups(backupsDir, 10 * TWENTY_FOUR_HOURS, 10)
       await createBackup(path)
+
+      console.log("====== CLEAR STATS:")
+      console.log("Last event id:", lastEventId)
+      console.log("Last trigger timestamp:", lastTriggerTimestamp)
+      console.log("Last command timestamp:", lastCommandTimestamp)
+
+      if(Number.isInteger(config.clearEvents) && lastEventId) {
+        const ts = (+(lastEventId.split(':')[0]) - config.clearEvents)
+        console.log("Clear events before", new Date(ts))
+        lastEventId = ts.toFixed(0).padStart(16, '0')+':'
+      }
+      if(Number.isInteger(config.clearTriggers) && lastTriggerTimestamp) {
+        const date = new Date(lastTriggerTimestamp)
+        const old = new Date(date.getTime() - config.clearTriggers)
+        console.log("Clear triggers before", old)
+        lastTriggerTimestamp = old.toISOString()
+      }
+      if(Number.isInteger(config.clearCommands) && lastCommandTimestamp) {
+        const date = new Date(lastCommandTimestamp)
+        const old = new Date(date.getTime() - config.clearCommands)
+        console.log("Clear commands before", old)
+        lastCommandTimestamp = old.toISOString()
+      }
+
+      console.log("====== CLEARING:")
+      console.log("Last remaining event id:", lastEventId)
+      console.log("Last remaining trigger timestamp:", lastTriggerTimestamp)
+      console.log("Last remaining command timestamp:", lastCommandTimestamp)
+
+      if(config.clearCommands) await removeOldCommands(lastCommandTimestamp, { delay: 10 })
+      if(config.clearTriggers) await removeOldTriggers(lastTriggerTimestamp, { delay: 10 })
       if(config.clearEvents) await removeOldEvents(lastEventId)
+      if(config.clearOpLogs) await removeOldOpLogs()
     })
   }
   currentBackup.promise.then(done => {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@live-change/backup-service",
-  "version": "0.8.137",
+  "version": "0.8.139",
   "description": "",
   "main": "index.js",
   "scripts": {
@@ -9,14 +9,14 @@
   "author": "Michał Łaszczewski <michal@emikse.com>",
   "license": "BSD-3-Clause",
   "dependencies": {
-    "@live-change/db-client": "^0.8.137",
-    "@live-change/framework": "^0.8.137",
+    "@live-change/db-client": "^0.8.139",
+    "@live-change/framework": "^0.8.139",
     "express": "^4.18.2",
     "express-basic-auth": "^1.2.1",
     "fs-extra": "^11.1.1",
     "p-queue": "^8.0.1",
     "progress-stream": "^2.0.0"
   },
-  "gitHead": "cb42255fbe7aa794dc5d7a6e57250ce8c959acf9",
+  "gitHead": "7f54396b91dc7b596380ec4d8b182c12c765fc6f",
   "type": "module"
 }

package/clearEvents.js

@@ -1,21 +0,0 @@
-import App from '@live-change/framework'
-const app = App.app()
-import definition from './definition.js'
-
-export async function getLastEventId() {
-  if(app.splitEvents) {
-    throw new Error("getLastEventId not supported in split events mode")
-  } else {
-    const lastLogs = app.dao.get(['database', 'logRange', app.databaseName, 'events', { reverse: true, limit: 1 }])
-    const lastLog = lastLogs[0]
-    return lastLog?.id || ''
-  }
-}
-
-export async function removeOldEvents(before) {
-  if(app.splitEvents) {
-    throw new Error("removeOldEvents not supported in split events mode")
-  } else {
-    await app.dao.request(['database', 'clearLog'], app.databaseName, 'events', before)
-  }
-}