mysql_db_tool 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: b7ba1c8784d5842dc7f1212c89ccaf8cd3fbd31793be8c3dd1c168bf75366ff7
+  data.tar.gz: e2eb44bd513921e7957f293076386ce63f614ee02bafbd242ee8f71bb8599145
+SHA512:
+  metadata.gz: fbe7207993d444b6cbf7c408f2860702bd0c63af6f09776865905598ab1c54aa7b4217e64921c5a742f6a1f7db1a19d4381d33aa8488eae2a31aa3fae0e03552
+  data.tar.gz: f90055728240fa5754b159b81e2798964956d299376c56eac7d31edad5e280483b739e6d3c73e47da5b11193c10f927625c503fe3bbff2437faded8d363a4f2b

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 VReRV
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,83 @@
+# MySQL Backup & Restore Tool
+
+[🇰🇷 (한국어)](./README_KO.md) | [🇬🇧 (English)](./README.md)
+
+A ruby script tool for backing up and restoring MySQL data.
+
+## Requirements
+
+* Tested on linux, Mac OS X environments only
+* ruby, mysql, mysqldump, gzip, zcat commands must be installed
+
+## Configuration
+
+Create a config-<env>.json file under the current directory according to the database environment.
+
+```json
+{
+  "dataTables": [],
+  "ignoreTables": [],
+  "dbInfo": {
+    "host": "localhost",
+    "port": 3306,
+    "user": "root",
+    "password": "",
+    "database": [
+      "test_db"
+    ]
+  }
+}
+```
+
+* dataTables - allows you to set a column for putting a --where condition to backup only the latest(3 days) rows of a large table.
+    * { "name": "table_name", "where": "column_name" }
+* ignoreTables - allows you to set which tables should be excluded from backups as unused tables.
+
+## Data backup
+
+```shell
+./bin/backup {env} {backup id} {run?} {gzip?}
+```
+
+* env - default (local), key to find the configuration file. e.g.) config-local.json
+* backup id - default (0), ID to use when restoring as a string
+* run? - Default (false), you can check in advance which command will be executed, if true, it will be executed
+* gzip? - default (true), whether to compress with gzip or not
+
+* DUMP_OPTIONS - you can set common mysqldump command options by this Environment variable,
+  if not set, it will use default options for mysqldump. (--single-transaction --skip-lock-tables)
+
+After execution, a directory named "backup-{backup id}" will be created under the current directory.
+
+## restore backup data
+
+```shell
+./bin/restore <env> {backup id} {run?} {drop all tables?}
+```
+
+* drop all tables? - Default (false), to keep existing tables, or true, which may cause integration check error if not set to true
+
+## Generate creating db and user sql
+
+You can generate a sql script to create a db and user.
+
+```shell
+./bin/gen_create_db_user {user} {password} {db} {host}
+```
+
+## Installing Ruby
+
+Install using `rbenv` on Mac OS X
+
+```bash
+brew install rbenv
+rbenv init
+rbenv install 3.3.0
+rbenv global 3.3.0
+ruby -v
+```
+
+## TODO
+
+* [ ] Add option to get db configuration information from spring cloud config
+* [ ] Change linux standard command line argument input format to --backup-id=1 when inputting arguments

data/bin/gen_create_db_user

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+
+require "mysql_db_tool"
+
+user=ARGV.count > 0 ? ARGV[0] : "db_user"
+password=ARGV.count > 1 ? ARGV[1] : "db_user_password"
+db=ARGV.count > 2 ? ARGV[2] : "db_name"
+host=ARGV.count > 3 ? ARGV[3] : "localhost"
+
+replace_map = {
+  "DB_USER" => user,
+  "DB_PASSWORD" => password,
+  "DB_NAME" => db,
+  "DB_HOST" => host
+}
+
+puts `cat #{MySQLDBTool.find_resource('sql/create_db_user_template.sql')} | ruby -pe '$_=$_#{replace_map.map { |k,v| ".gsub(/#{k}/, \"#{v}\")" }.join("")}' `

data/bin/mysql_backup

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "mysql_db_tool"
+
+# Parse command line arguments
+options = {}
+options[:env] = ARGV[0] || "local"
+options[:id] = ARGV[1] || "0"
+options[:run] = ARGV[2] == "true"
+options[:gzip] = ARGV.count > 3 ? ARGV[3] == "true" : true
+
+# Print the options for verification
+puts "options=#{options}"
+
+# Perform the backup
+# begin
+  MySQLDBTool.backup(options)
+# rescue => e
+#   puts "Error during backup: #{e.message}"
+#   exit 1
+# end
+
+puts "Backup completed successfully."

data/bin/mysql_restore

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "mysql_db_tool"
+
+# Parse command line arguments
+options = {}
+options[:env] = ARGV[0] || "local"
+options[:id] = ARGV[1] || "0"
+options[:run] = ARGV[2] == "true"
+options[:drop_all_tables] = ARGV.count > 3 ? ARGV[3] == "true" : true
+
+# Print the options for verification
+puts "Env: #{options[:env]}"
+puts "Backup ID: #{options[:id]}"
+puts "Run: #{options[:run]}"
+puts "DropAllTables: #{options[:drop_all_tables]}"
+
+# Perform the backup
+# begin
+  MySQLDBTool.restore(options)
+# rescue => e
+#   puts "Error during backup: #{e.message}"
+#   exit 1
+# end
+
+puts "Restore completed successfully."

data/lib/mysql_db_tool/backup.rb

@@ -0,0 +1,70 @@
+require 'date'
+require_relative "db_backup_base"
+require_relative "config/config_loader"
+
+module MySQLDBTool
+
+  class Backup
+
+    def initialize(options = {})
+      @options = options
+      tableConfig = MySQLDBTool::Config::ConfigLoader.load(options[:env])
+      @data_tables = tableConfig[:data_tables]
+      @ignore_tables = tableConfig[:ignore_tables]
+      @db_info = tableConfig[:db_info]
+    end
+
+    def perform
+
+      verify_tools_exist
+
+      id=@options[:id] || "0"
+      isGzip=@options[:gzip]
+      limitDays=@options[:limit_days] || 3
+
+      Array(@db_info[:database]).flat_map.each_with_index do |database, index |
+
+        defaultOptions="--column-statistics=0 #{mysqlDefaultOptions(@db_info, database)}"
+        backupDir=backupDirName(id, "#{index}_#{database}")
+
+        commands = []
+        if File.exist? backupDir
+          puts "[skip - directory exists] #{backupDir}"
+          return commands
+        end
+        commands.push "mkdir -p #{backupDir}"
+
+        backupFile="#{backupDir}/#{DateTime.now.strftime("%Y-%m-%d")}_#{id}"
+        whereDate=(Date.today - limitDays).strftime("%Y-%m-%d 00:00:00")
+        options=ENV['DUMP_OPTIONS'] || "--single-transaction --skip-lock-tables"
+
+        puts "backupFile=#{backupFile}"
+
+        ignoreTablesOption = @ignore_tables.map { |e| "--ignore-table=#{@db_info[:database]}.#{e}"  }.join(' ')
+
+        commands.push gzipCommand("mysqldump --no-data #{ignoreTablesOption} #{defaultOptions}", isGzip, "#{backupFile}-schema.sql")
+
+        backupTables = []
+
+        @data_tables.each {|table|
+          where = table[:where].empty? ? "" : "--where=\"#{table[:where]} >= '#{whereDate}'\""
+          if where.empty?
+            backupTables.push(table[:name])
+            next
+          else
+            commands.push(gzipCommand("mysqldump --no-create-info #{options} #{where} #{defaultOptions} #{table[:name]}", isGzip, "#{backupFile}-#{table[:name]}.sql"))
+          end
+        }
+
+        commands.push(gzipCommand("mysqldump --no-create-info #{options} #{defaultOptions} #{backupTables.join(' ')}", isGzip, "#{backupFile}-all-other-tables.sql"))
+        commands
+      end
+    end
+
+    def gzipCommand(command, isGzip, file)
+      "#{command} #{isGzip ? '| gzip ' : ''} > #{file}"
+    end
+
+  end
+
+end

data/lib/mysql_db_tool/config/config_loader.rb

@@ -0,0 +1,47 @@
+require 'json'
+
+module MySQLDBTool
+  module Config
+    class ConfigLoader
+      DEFAULT_CONFIG = {
+        db_info: {
+          host: "localhost",
+          user: "root",
+          password: "",
+          database: ["mysql"],
+          port: 3306
+        },
+        data_tables: [
+          # { name: "large_table1", where: "updated_at" }
+        ],
+        ignore_tables: [
+        ]
+      }
+
+      def self.load(environment)
+        file_path = File.join(Dir.pwd, "config-#{environment}.json")
+        if File.exist?(file_path)
+          file_contents = File.read(file_path)
+          json_data = JSON.parse(file_contents)
+          {
+            db_info: symbolize_keys(json_data['dbInfo']),
+            data_tables: json_data['dataTables'].map { |table| symbolize_keys(table) },
+            ignore_tables: json_data['ignoreTables']
+          }
+        else
+          puts "Warning: config-#{environment}.json not found in the current directory. Using default configuration."
+          DEFAULT_CONFIG
+        end
+      rescue JSON::ParserError => e
+        puts "Error parsing config-#{environment}.json: #{e.message}. Using default configuration."
+        DEFAULT_CONFIG
+      end
+
+      private
+
+      def self.symbolize_keys(hash)
+        hash.transform_keys(&:to_sym)
+      end
+    end
+  end
+end

data/lib/mysql_db_tool/db_backup_base.rb

@@ -0,0 +1,38 @@
+
+
+def run(isRun, command)
+  if not isRun
+    puts "[dryRun] #{command}"
+  else
+    puts "Running: [#{command}]"
+    puts `#{command}`
+  end
+end
+
+def backupDirName(id, dbName = "")
+  "backup-#{id}#{dbName.empty? ? '' : "/#{dbName}"}"
+end
+
+def mysqlDefaultOptions(db_info, database)
+  " --ssl-mode=disabled -h #{db_info[:host]} -u #{db_info[:user]} #{db_info[:password].to_s.empty? ? '' : " -p'#{db_info[:password]}'"} #{db_info[:port].to_s.empty? ? '' : " -P'#{db_info[:port]}'"} #{database} "
+end
+
+def verify_tools_exist
+  tools = ["mysql", "mysqldump", "gzip", "zcat"]
+  missing_tools = []
+
+  tools.each do |tool|
+    if not system("which #{tool} > /dev/null 2>&1")
+      missing_tools << tool
+      puts "'#{tool}' is not available"
+    end
+  end
+
+  if missing_tools.empty?
+    puts "All required tools are available."
+  else
+    puts "Missing tools: #{missing_tools.join(', ')}"
+    puts "Please install the missing tools."
+    exit 1
+  end
+end

data/lib/mysql_db_tool/restore.rb

@@ -0,0 +1,101 @@
+require_relative "db_backup_base"
+
+module MySQLDBTool
+
+  class Restore
+
+    def initialize(options = {})
+      @options = options
+      tableConfig = MySQLDBTool::Config::ConfigLoader.load(options[:env])
+      @data_tables = tableConfig[:data_tables]
+      @ignore_tables = tableConfig[:ignore_tables]
+      @db_info = tableConfig[:db_info]
+    end
+
+    def perform
+
+      verify_tools_exist
+
+      env=@options[:env] || "local"
+
+      if env.downcase.end_with? "prod"
+        abort "Aborted. Never restore production DB!"
+      end
+
+      id=@options[:id] || "0"
+      isRun=@options[:run]
+      isDropAllTables=@options[:drop_all_tables]
+
+      puts "ARGV=#{ARGV}, env=#{env}, id=#{id}, run=#{isRun} isDropAllTables=#{isDropAllTables}"
+
+      backupDir=backupDirName(id)
+
+      puts "backupDir=#{backupDir}"
+      databases = Array(@db_info[:database])
+
+      databaseMap = {}
+      sameDb = true
+
+      Dir.entries(backupDir).reject {|f| File.directory? f}.sort.each do |f|
+
+        index, origin_database = split_integer_and_string(f)
+        database = get_element_or_last(databases, index)
+        sameDb = sameDb && (database == origin_database)
+        databaseMap["`#{origin_database}`\\."] = "`#{database}`."
+      end
+
+      gsubstring = sameDb ? "" : databaseMap.map { |k,v| ".gsub(/#{k}/, \"#{v}\")" }.join("")
+
+      commands = []
+
+      Dir.entries(backupDir).reject {|f| File.directory? f}.sort.flat_map do |f|
+
+        index, origin_database = split_integer_and_string(f)
+        database = get_element_or_last(databases, index)
+
+        defaultOptions=mysqlDefaultOptions(@db_info, database)
+        backupDir=backupDirName(id, f)
+
+        commands.push("cat #{MySQLDBTool.find_resource('sql/drop_all_tables.sql')} | mysql #{defaultOptions}") if isDropAllTables
+        commands.push("cat #{MySQLDBTool.find_resource('sql/drop_all_views.sql')} | mysql #{defaultOptions}") if isDropAllTables
+
+        Dir.entries(backupDir).reject {|f| File.directory? f} .select {|f| f.include?("-schema.sql")} .each {|f|
+          restore_each(commands, backupDir+"/"+f, defaultOptions, gsubstring)
+        }
+        Dir.entries(backupDir).reject {|f| File.directory? f} .reject {|f| f.include?("-schema.sql")} .each {|f|
+          restore_each(commands, backupDir+"/"+f, defaultOptions, gsubstring)
+        }
+        commands
+      end
+    end
+
+    private
+
+    def restore_each(commands, file, defaultOptions, gsubstring)
+      command = ""
+      replacing = " | ruby -pe '$_=$_#{gsubstring}'" unless gsubstring.empty?
+      if file.end_with? ".sql.gz"
+        command = "zcat #{file} #{replacing} | mysql #{defaultOptions}"
+      elsif file.end_with? ".sql"
+        command = "cat #{file} #{replacing} | mysql #{defaultOptions}"
+      else
+        puts "not supported file #{file}"
+      end
+
+      commands.push(command) if not command.empty?
+    end
+
+    def split_integer_and_string(input)
+      parts = input.split('_', 2)
+      integer = parts[0].to_i
+      string = parts[1]
+      [integer, string]
+    end
+
+    def get_element_or_last(array, index)
+      index < array.length ? array[index] : array.last
+    end
+
+  end
+end
+

data/lib/mysql_db_tool/sql/create_db_user_template.sql

@@ -0,0 +1,9 @@
+CREATE DATABASE DB_NAME;
+
+CREATE USER 'DB_USER'@'DB_HOST' IDENTIFIED WITH mysql_native_password BY 'DB_PASSWORD';
+GRANT SELECT, INSERT, UPDATE, DELETE, CREATE, INDEX, DROP, ALTER, REFERENCES, CREATE TEMPORARY TABLES, LOCK TABLES ON DB_NAME.* TO 'DB_USER'@'DB_HOST';
+GRANT SELECT, CREATE VIEW, SHOW VIEW, LOCK TABLES, EVENT, TRIGGER ON DB_NAME.* TO 'DB_USER'@'DB_HOST';
+-- privileges for mysqldump command
+GRANT FILE, RELOAD, REPLICATION CLIENT, SESSION_VARIABLES_ADMIN, SYSTEM_VARIABLES_ADMIN ON *.* TO 'DB_USER'@'DB_HOST';
+
+FLUSH PRIVILEGES;

data/lib/mysql_db_tool/sql/drop_all_tables.sql

@@ -0,0 +1,14 @@
+SET FOREIGN_KEY_CHECKS = 0;
+SET GROUP_CONCAT_MAX_LEN=32768;
+SET @tables = NULL;
+SELECT GROUP_CONCAT('`', table_name, '`') INTO @tables
+  FROM information_schema.tables
+  WHERE table_schema = (SELECT DATABASE());
+SELECT IFNULL(@tables,'dummy') INTO @tables;
+
+SET @tables = CONCAT('DROP TABLE IF EXISTS ', @tables);
+PREPARE stmt FROM @tables;
+EXECUTE stmt;
+DEALLOCATE PREPARE stmt;
+SET FOREIGN_KEY_CHECKS = 1;
+

data/lib/mysql_db_tool/sql/drop_all_views.sql

@@ -0,0 +1,14 @@
+SET FOREIGN_KEY_CHECKS = 0;
+SET GROUP_CONCAT_MAX_LEN=32768;
+SET @tables = NULL;
+SELECT GROUP_CONCAT('`', table_name, '`') INTO @tables
+  FROM information_schema.views
+  WHERE table_schema = (SELECT DATABASE());
+SELECT IFNULL(@tables,'dummy') INTO @tables;
+
+SET @tables = CONCAT('DROP VIEW IF EXISTS ', @tables);
+PREPARE stmt FROM @tables;
+EXECUTE stmt;
+DEALLOCATE PREPARE stmt;
+SET FOREIGN_KEY_CHECKS = 1;
+

data/lib/mysql_db_tool/version.rb

@@ -0,0 +1,5 @@
+# lib/mysql_db_tool/version.rb
+
+module MySQLDBTool
+  VERSION = "0.1.0"
+end

data/lib/mysql_db_tool.rb

@@ -0,0 +1,29 @@
+# Require all necessary files
+require_relative 'mysql_db_tool/backup'
+require_relative 'mysql_db_tool/restore'
+require_relative 'mysql_db_tool/db_backup_base'
+
+# Define the main module
+module MySQLDBTool
+  class Error < StandardError; end
+
+  # You might want to add methods to easily access your main functionalities
+  def self.backup(options = {})
+    isRun = options[:run]
+    commands = Backup.new(options).perform
+    commands.each { |command| run(isRun, command) }
+  end
+
+  def self.restore(options = {})
+    isRun = options[:run]
+    commands = Restore.new(options).perform
+    commands.each { |command| run(isRun, command) }
+  end
+
+  def self.find_resource(relative_path)
+    File.join(File.dirname(__FILE__), 'mysql_db_tool', relative_path)
+  end
+end
+
+# You might want to include a version file
+require_relative 'mysql_db_tool/version'

metadata

@@ -0,0 +1,102 @@
+--- !ruby/object:Gem::Specification
+name: mysql_db_tool
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Soonoh Jung
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2024-08-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '13.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: A Ruby gem for backing up and restoring MySQL databases
+email:
+- soonoh.jung@vrerv.com
+executables:
+- mysql_backup
+- mysql_restore
+- gen_create_db_user
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- bin/gen_create_db_user
+- bin/mysql_backup
+- bin/mysql_restore
+- lib/mysql_db_tool.rb
+- lib/mysql_db_tool/backup.rb
+- lib/mysql_db_tool/config/config_loader.rb
+- lib/mysql_db_tool/db_backup_base.rb
+- lib/mysql_db_tool/restore.rb
+- lib/mysql_db_tool/sql/create_db_user_template.sql
+- lib/mysql_db_tool/sql/drop_all_tables.sql
+- lib/mysql_db_tool/sql/drop_all_views.sql
+- lib/mysql_db_tool/version.rb
+homepage: https://github.com/vrerv/mysql-db-tool
+licenses:
+- MIT
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.5.17
+signing_key:
+specification_version: 4
+summary: MySQL DB Tool
+test_files: []