electerm-data-tool 1.0.0

package/src/migrate/index.js

@@ -0,0 +1,85 @@
+/**
+ * common data upgrade process
+ * It will check current version in db and check version in package.json,
+ * run every upgrade script one by one
+ */
+
+const { packInfo } = require('../common/app-props')
+const { version: packVersion } = packInfo
+const { resolve } = require('path')
+const fs = require('fs')
+const log = require('../common/log')
+const compare = require('../common/version-compare')
+const { dbAction } = require('../nedb')
+const _ = require('lodash')
+const { updateDBVersion } = require('./version-upgrade')
+const emptyVersion = '0.0.0'
+const versionQuery = {
+  _id: 'version'
+}
+
+async function getDBVersion () {
+  const version = await dbAction('data', 'findOne', versionQuery)
+    .then(doc => {
+      return doc ? doc.value : emptyVersion
+    })
+    .catch(e => {
+      log.error(e)
+      return emptyVersion
+    })
+  return version
+}
+
+/**
+ * get upgrade versions should be run as version upgrade
+ */
+async function getUpgradeVersionList () {
+  const version = await getDBVersion()
+  const list = fs.readdirSync(__dirname)
+  return list.filter(f => {
+    const vv = f.replace('.js', '').replace('v', '')
+    return /^v\d/.test(f) && compare(vv, version) > 0 && compare(vv, packVersion) <= 0
+  }).sort((a, b) => {
+    return compare(a, b)
+  })
+}
+async function versionShouldUpgrade () {
+  const dbVersion = await getDBVersion()
+  log.info('database version:', dbVersion)
+  return compare(dbVersion, packVersion) < 0
+}
+
+async function shouldUpgrade () {
+  const shouldUpgradeVersion = await versionShouldUpgrade()
+  if (!shouldUpgradeVersion) {
+    return false
+  }
+  const dbVersion = await getDBVersion()
+  log.info('dbVersion', dbVersion)
+  if (dbVersion === emptyVersion) {
+    return false
+  }
+  const list = await getUpgradeVersionList()
+  if (_.isEmpty(list)) {
+    await updateDBVersion(packVersion)
+    return false
+  }
+  return {
+    dbVersion,
+    packVersion
+  }
+}
+
+async function doUpgrade () {
+  const list = await getUpgradeVersionList()
+  log.info('Upgrading...')
+  for (const v of list) {
+    const p = resolve(__dirname, v)
+    const run = require(p)
+    await run()
+  }
+  log.info('Upgrade end')
+}
+
+exports.checkDbUpgrade = shouldUpgrade
+exports.doUpgrade = doUpgrade

package/src/migrate/migrate-1-to-2.js

@@ -0,0 +1,117 @@
+/**
+ * migrate from NeDB (v1) to SQLite (v2)
+ */
+
+const { resolve } = require('path')
+const { existsSync, renameSync } = require('fs')
+const { appPath, defaultUserName } = require('../common/app-props')
+const log = require('../common/log')
+
+const reso = (name) => {
+  return resolve(appPath, 'electerm', 'users', defaultUserName, `electerm.${name}.nedb`)
+}
+
+const tables = [
+  'bookmarks',
+  'bookmarkGroups',
+  'addressBookmarks',
+  'terminalThemes',
+  'lastStates',
+  'data',
+  'quickCommands',
+  'log',
+  'dbUpgradeLog',
+  'profiles'
+]
+
+/**
+ * Check if migration from v1 (NeDB) to v2 (SQLite) is needed
+ * @returns {boolean} true if migration is needed
+ */
+function checkMigrate () {
+  if (process.versions.node < '22.0.0') {
+    return false
+  }
+  // Check if any NeDB files exist
+  for (const table of tables) {
+    const nedbPath = reso(table)
+    if (existsSync(nedbPath)) {
+      return true
+    }
+  }
+  return false
+}
+
+/**
+ * Migrate all data from NeDB to SQLite and backup NeDB files
+ */
+async function migrate () {
+  log.info('Starting migration from NeDB (v1) to SQLite (v2)...')
+  // Import both database modules
+  const nedbModule = require('../nedb')
+  const sqliteModule = require('../sqlite')
+  const {
+    checkDbUpgrade,
+    doUpgrade
+  } = require('./index')
+  if (await checkDbUpgrade()) {
+    await doUpgrade()
+  }
+
+  const timeStamp = Date.now()
+
+  // Migrate data from each table
+  for (const table of tables) {
+    const nedbPath = reso(table)
+
+    if (existsSync(nedbPath)) {
+      log.info(`Migrating table: ${table}`)
+
+      // Read all data from NeDB
+      const nedbData = await nedbModule.dbAction(table, 'find', {})
+
+      if (nedbData && nedbData.length > 0) {
+        log.info(`Found ${nedbData.length} records in ${table}`)
+
+        // Insert/update data into SQLite using upsert to avoid conflicts
+        for (const record of nedbData) {
+          // Ensure record has an _id field
+          const recordId = record._id || record.id
+          if (!recordId) {
+            log.warn(`Record in ${table} has no _id or id field, skipping:`, record)
+            continue
+          }
+          // Use update with upsert option to handle existing records gracefully
+          await sqliteModule.dbAction(table, 'update',
+            { _id: recordId },
+            { $set: record },
+            { upsert: true }
+          )
+        }
+
+        log.info(`Successfully migrated ${nedbData.length} records from ${table}`)
+      } else {
+        log.info(`Table ${table} is empty, nothing to migrate`)
+      }
+
+      // Rename NeDB file to .bak
+      const backupPath = `${nedbPath}-${timeStamp}.bak`
+      try {
+        renameSync(nedbPath, backupPath)
+        log.info(`Backed up ${nedbPath} to ${backupPath}`)
+      } catch (renameError) {
+        log.error(`Error backing up ${nedbPath}:`, renameError)
+      }
+    } else {
+      log.info(`NeDB file for ${table} does not exist, skipping`)
+    }
+  }
+
+  log.info('Migration from NeDB to SQLite completed successfully')
+  return true
+}
+
+module.exports = {
+  checkMigrate,
+  migrate
+}

package/src/migrate/v1.25.0.js

@@ -0,0 +1,48 @@
+/**
+ * upgrade database to v1.7.0
+ */
+
+const { userConfigId } = require('../common/constants')
+const { dbAction } = require('../lib/nedb')
+const { updateDBVersion } = require('./version-upgrade')
+const log = require('../common/log')
+
+async function fixAll () {
+  const q = {
+    _id: userConfigId
+  }
+  const conf = await dbAction('data', 'findOne', q)
+  if (!conf) {
+    return
+  }
+  if (!conf.syncSetting) {
+    conf.syncSetting = {}
+  }
+  const {
+    syncEncrypt,
+    githubAccessToken,
+    giteeAccessToken
+  } = conf.syncSetting
+  if (!syncEncrypt) {
+    return
+  }
+  if (githubAccessToken) {
+    conf.syncSetting.githubSyncPassword = githubAccessToken
+  }
+  if (giteeAccessToken) {
+    conf.syncSetting.giteeSyncPassword = giteeAccessToken
+  }
+  delete conf.syncSetting.syncEncrypt
+  await dbAction('data', 'update', q, {
+    ...q,
+    ...conf
+  })
+}
+
+module.exports = async () => {
+  const versionTo = '1.25.0'
+  log.info(`Start: upgrading to v${versionTo}`)
+  await fixAll()
+  await updateDBVersion(versionTo)
+  log.info(`Done: upgrading to v${versionTo}`)
+}

package/src/migrate/v1.27.17.js

@@ -0,0 +1,66 @@
+/**
+ * upgrade database to v1.27.7
+ */
+
+const { userConfigId } = require('../common/constants')
+const { dbAction } = require('../lib/nedb')
+const { updateDBVersion } = require('./version-upgrade')
+const log = require('../common/log')
+const { buildProxyString } = require('../lib/build-proxy')
+
+async function fixConf () {
+  log.info('Start update global proxy config')
+  const q = {
+    _id: userConfigId
+  }
+  const conf = await dbAction('data', 'findOne', q)
+  if (!conf) {
+    return
+  }
+  const proxy = buildProxyString(conf)
+  if (proxy) {
+    conf.proxy = proxy
+  }
+  const props = [
+    'proxyPort', 'proxyType', 'proxyIp', 'proxyUsername', 'proxyPassword'
+  ]
+  for (const p of props) {
+    delete conf[p]
+  }
+  await dbAction('data', 'update', q, {
+    ...q,
+    ...conf
+  })
+}
+
+async function fixBookmarks () {
+  log.info('Start update bookmark proxy config')
+  const arr = await dbAction('bookmarks', 'find', {})
+  const len = arr.length
+  let i = 0
+  log.info('bookmarks count:', len)
+  for (const b of arr) {
+    const proxy = buildProxyString(b.proxy || {})
+    console.log(i + 1, b._id, proxy)
+    await dbAction('bookmarks', 'update', {
+      _id: b._id
+    }, {
+      ...b,
+      proxy
+    })
+    i = i + 1
+  }
+}
+
+async function fixAll () {
+  await fixConf()
+  await fixBookmarks()
+}
+
+module.exports = async () => {
+  const versionTo = '1.27.17'
+  log.info(`Start: upgrading to v${versionTo}`)
+  await fixAll()
+  await updateDBVersion(versionTo)
+  log.info(`Done: upgrading to v${versionTo}`)
+}

package/src/migrate/v1.3.9.js

@@ -0,0 +1,44 @@
+/**
+ * upgrade database to v1.3.9
+ * migrate old file based db to nedb
+ */
+
+const { dbAction, tables } = require('../lib/nedb')
+const { updateDBVersion } = require('./version-upgrade')
+const log = require('../common/log')
+
+function wait (time) {
+  return new Promise(resolve => {
+    setTimeout(resolve, time)
+  })
+}
+
+async function fixAll () {
+  for (const name of tables) {
+    const all = await dbAction(name, 'find', {})
+    for (const inst of all) {
+      const { id, _id, ...rest } = inst
+      if (id) {
+        await dbAction(name, 'remove', {
+          id
+        })
+        await dbAction(name, 'remove', {
+          _id: id
+        }, { multi: true })
+        await wait(100)
+        await dbAction(name, 'insert', {
+          _id: id,
+          ...rest
+        })
+      }
+    }
+  }
+}
+
+module.exports = async () => {
+  const versionTo = '1.3.9'
+  log.info(`Start: upgrading to v${versionTo}`)
+  await fixAll()
+  await updateDBVersion(versionTo)
+  log.info(`Done: upgrading to v${versionTo}`)
+}

package/src/migrate/v1.32.36.js

@@ -0,0 +1,44 @@
+/**
+ * upgrade database to v1.32.36
+ */
+
+const { dbAction } = require('../lib/nedb')
+const { updateDBVersion } = require('./version-upgrade')
+const log = require('../common/log')
+const { buildSshTunnels } = require('../common/build-ssh-tunnel')
+
+async function fixBookmarks () {
+  log.info('Start update bookmark Ssh Tunnels config')
+  const arr = await dbAction('bookmarks', 'find', {})
+  const len = arr.length
+  let i = 0
+  log.info('bookmarks count:', len)
+  for (const b of arr) {
+    console.log(i + 1, b._id, b.sshTunnel ? 'has sshTunnel' : 'no sshTunnel')
+    if (b.sshTunnel) {
+      const sshTunnels = buildSshTunnels(b)
+      delete b.sshTunnel
+      delete b.sshTunnelRemotePort
+      delete b.sshTunnelLocalPort
+      await dbAction('bookmarks', 'update', {
+        _id: b._id
+      }, {
+        ...b,
+        sshTunnels
+      })
+    }
+    i = i + 1
+  }
+}
+
+async function fixAll () {
+  await fixBookmarks()
+}
+
+module.exports = async () => {
+  const versionTo = '1.32.36'
+  log.info(`Start: upgrading to v${versionTo}`)
+  await fixAll()
+  await updateDBVersion(versionTo)
+  log.info(`Done: upgrading to v${versionTo}`)
+}

package/src/migrate/v1.34.20.js

@@ -0,0 +1,43 @@
+/**
+ * upgrade database to v1.34.20
+ */
+
+const { dbAction } = require('../lib/nedb')
+const { updateDBVersion } = require('./version-upgrade')
+const log = require('../common/log')
+const { buildRunScripts } = require('../common/build-run-scripts')
+
+async function fixBookmarks () {
+  log.info('Start update bookmark loginScript config')
+  const arr = await dbAction('bookmarks', 'find', {})
+  const len = arr.length
+  let i = 0
+  log.info('bookmarks count:', len)
+  for (const b of arr) {
+    console.log(i + 1, b._id, b.loginScript ? 'has loginScript' : 'no loginScript')
+    if (b.loginScript) {
+      const runScripts = buildRunScripts(b)
+      delete b.loginScript
+      delete b.loginScriptDelay
+      await dbAction('bookmarks', 'update', {
+        _id: b._id
+      }, {
+        ...b,
+        runScripts
+      })
+    }
+    i = i + 1
+  }
+}
+
+async function fixAll () {
+  await fixBookmarks()
+}
+
+module.exports = async () => {
+  const versionTo = '1.34.20'
+  log.info(`Start: upgrading to v${versionTo}`)
+  await fixAll()
+  await updateDBVersion(versionTo)
+  log.info(`Done: upgrading to v${versionTo}`)
+}

package/src/migrate/v1.34.59.js

@@ -0,0 +1,32 @@
+/**
+ * upgrade database to v1.34.20
+ */
+
+const { dbAction } = require('../lib/nedb')
+const { userConfigId } = require('../common/constants')
+const { updateDBVersion } = require('./version-upgrade')
+const log = require('../common/log')
+
+async function fixAll () {
+  log.info('Start update default terminal word separator config')
+  const q = {
+    _id: userConfigId
+  }
+  const conf = await dbAction('data', 'findOne', q)
+  if (conf && conf.terminalWordSeparator && !conf.terminalWordSeparator.includes(' ')) {
+    conf.terminalWordSeparator = conf.terminalWordSeparator.slice(0, 1) + ' ' +
+      conf.terminalWordSeparator.slice(1)
+    await dbAction('data', 'update', q, {
+      ...q,
+      ...conf
+    })
+  }
+}
+
+module.exports = async () => {
+  const versionTo = '1.34.59'
+  log.info(`Start: upgrading to v${versionTo}`)
+  await fixAll()
+  await updateDBVersion(versionTo)
+  log.info(`Done: upgrading to v${versionTo}`)
+}

package/src/migrate/v1.5.13.js

@@ -0,0 +1,57 @@
+/**
+ * upgrade database to v1.5.13
+ */
+
+const { userConfigId } = require('../common/constants')
+const { dbAction } = require('../lib/nedb')
+const { updateDBVersion } = require('./version-upgrade')
+const log = require('../common/log')
+const { decrypt } = require('../lib/enc')
+
+async function fixAll () {
+  const q = {
+    _id: userConfigId
+  }
+  const conf = await dbAction('data', 'findOne', q)
+  if (!conf.syncSetting) {
+    conf.syncSetting = {}
+  }
+  const {
+    encrypted,
+    gistId,
+    githubAccessToken,
+    lastSyncTime
+  } = conf.syncSetting
+  if (lastSyncTime) {
+    conf.syncSetting.githubLastSyncTime = lastSyncTime
+  }
+  if (gistId) {
+    conf.syncSetting.githubGistId = gistId
+  }
+  if (
+    encrypted &&
+    gistId &&
+    githubAccessToken
+  ) {
+    const nt = decrypt(
+      conf.syncSetting.githubAccessToken,
+      conf.syncSetting.gistId
+    )
+    conf.syncSetting.githubAccessToken = nt
+  }
+  delete conf.syncSetting.encrypted
+  delete conf.syncSetting.lastSyncTime
+  delete conf.syncSetting.gistId
+  await dbAction('data', 'update', q, {
+    ...q,
+    ...conf
+  })
+}
+
+module.exports = async () => {
+  const versionTo = '1.5.13'
+  log.info(`Start: upgrading to v${versionTo}`)
+  await fixAll()
+  await updateDBVersion(versionTo)
+  log.info(`Done: upgrading to v${versionTo}`)
+}

package/src/migrate/v1.7.0.js

@@ -0,0 +1,36 @@
+/**
+ * upgrade database to v1.7.0
+ */
+
+const { dbAction } = require('../lib/nedb')
+const { updateDBVersion } = require('./version-upgrade')
+const log = require('../common/log')
+const defaults = require('./db-defaults')
+
+async function fixAll () {
+  const defaultThemeConfig = defaults[0].data[0]
+  const defaultLightThemeConfig = defaults[0].data[1]
+  const all = await dbAction('terminalThemes', 'find', {}).catch(log.error) || []
+  for (const item of all) {
+    const q = {
+      _id: item.id || item._id
+    }
+    const updates = q._id === 'default'
+      ? defaultThemeConfig
+      : {
+          ...item,
+          uiThemeConfig: defaultThemeConfig.uiThemeConfig
+        }
+    await dbAction('terminalThemes', 'update', q, updates).catch(log.error)
+  }
+  await dbAction('terminalThemes', 'insert', defaultLightThemeConfig).catch(log.error)
+  log.info('end: update db')
+}
+
+module.exports = async () => {
+  const versionTo = '1.7.0'
+  log.info(`Start: upgrading to v${versionTo}`)
+  await fixAll()
+  await updateDBVersion(versionTo)
+  log.info(`Done: upgrading to v${versionTo}`)
+}

package/src/migrate/version-upgrade.js

@@ -0,0 +1,39 @@
+/**
+ * upgrade db version
+ */
+
+/**
+ * common data upgrade process
+ * It will check current version in db and check version in package.json,
+ * run every upgrade script one by one
+ */
+
+const log = require('../common/log')
+const { dbAction } = require('../nedb')
+
+async function updateDBVersion (toVersion) {
+  const versionQuery = {
+    _id: 'version'
+  }
+  log.info('upgrade db version to', toVersion)
+  await dbAction('data', 'update', versionQuery, {
+    ...versionQuery,
+    value: toVersion
+  }, {
+    upsert: true
+  })
+    .catch(e => {
+      log.error(e)
+      log.error('upgrade db version error', toVersion)
+    })
+  await dbAction('dbUpgradeLog', 'insert', {
+    time: Date.now(),
+    toVersion
+  })
+    .catch(e => {
+      log.error(e)
+      log.error('insert dbUpgradeLog error', toVersion)
+    })
+}
+
+exports.updateDBVersion = updateDBVersion

package/src/nedb.js

@@ -0,0 +1,52 @@
+/**
+ * nedb api wrapper
+ */
+
+const { appPath, defaultUserName } = require('./common/app-props')
+const { resolve } = require('path')
+const Datastore = require('@yetzt/nedb')
+const db = {}
+
+const reso = (name) => {
+  return resolve(appPath, 'electerm', 'users', defaultUserName, `electerm.${name}.nedb`)
+}
+const tables = [
+  'bookmarks',
+  'bookmarkGroups',
+  'addressBookmarks',
+  'terminalThemes',
+  'lastStates',
+  'data',
+  'quickCommands',
+  'log',
+  'dbUpgradeLog',
+  'profiles'
+]
+
+tables.forEach(table => {
+  const conf = {
+    filename: reso(table),
+    autoload: true
+  }
+  db[table] = new Datastore(conf)
+})
+
+const dbAction = (dbName, op, ...args) => {
+  if (op === 'compactDatafile') {
+    db[dbName].persistence.compactDatafile()
+    return
+  }
+  return new Promise((resolve, reject) => {
+    db[dbName][op](...args, (err, result) => {
+      if (err) {
+        return reject(err)
+      }
+      resolve(result)
+    })
+  })
+}
+
+module.exports = {
+  dbAction,
+  tables
+}