sg_tiny_backup 1.0.0

data/lib/sg_tiny_backup/commands/base.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module SgTinyBackup
+  module Commands
+    class Base
+      def command
+        raise NotImplementedError
+      end
+
+      def env
+        {}
+      end
+
+      def success_codes
+        [0]
+      end
+
+      def warning_message
+        nil
+      end
+    end
+  end
+end

data/lib/sg_tiny_backup/commands/gzip.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module SgTinyBackup
+  module Commands
+    class Gzip < Base
+      def initialize(level: nil)
+        super()
+        @level = level
+      end
+
+      def command
+        parts = ["gzip"]
+        parts << "-#{@level}" if @level
+        parts.join(" ")
+      end
+    end
+  end
+end

data/lib/sg_tiny_backup/commands/mysql_dump.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module SgTinyBackup
+  module Commands
+    class MysqlDump < Base
+      attr_reader :user, :host, :port, :password, :database, :extra_options
+
+      def initialize(database:, host: nil, port: nil, user: nil, password: nil, extra_options: nil) # rubocop:disable Metrics/ParameterLists
+        super()
+        @user = user
+        @host = host
+        @port = port
+        @password = password
+        @database = database
+        @extra_options = extra_options
+      end
+
+      def command
+        parts = []
+        parts << "mysqldump"
+        parts << extra_options if extra_options
+        parts << "--user=#{Shellwords.escape(@user)}" if @user
+        parts << "--host=#{@host}" if @host
+        parts << "--port=#{@port}" if @port
+        parts << @database
+        parts.join(" ")
+      end
+
+      def env
+        {
+          "MYSQL_PWD" => @password,
+        }.compact
+      end
+    end
+  end
+end

data/lib/sg_tiny_backup/commands/openssl.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module SgTinyBackup
+  module Commands
+    class Openssl < Base
+      CIPHER = "aes-256-cbc"
+      ITER = 10_000
+
+      def initialize(password:, filename: nil)
+        super()
+        @password = password
+        @filename = filename
+      end
+
+      def command
+        parts = ["openssl enc -#{CIPHER} -pbkdf2 -iter #{ITER}"]
+        parts << "-pass env:SG_TINY_BACKUP_ENCRYPTION_KEY"
+        parts << "-out #{@filename}" if @filename
+        parts.join(" ")
+      end
+
+      def env
+        {
+          "SG_TINY_BACKUP_ENCRYPTION_KEY" => @password,
+        }
+      end
+
+      class << self
+        def decryption_command
+          parts = ["openssl enc -d -#{CIPHER} -pbkdf2 -iter #{ITER}"]
+          parts << "-pass pass:ENCRYPTION_KEY"
+          parts << "-in INPUTFILE -out OUTPUTFILE"
+          parts.join(" ")
+        end
+      end
+    end
+  end
+end

data/lib/sg_tiny_backup/commands/pg_dump.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+require_relative "base"
+
+module SgTinyBackup
+  module Commands
+    class PgDump < Base
+      attr_reader :user, :host, :port, :password, :database, :extra_options
+
+      def initialize(database:, host: nil, port: nil, user: nil, password: nil, extra_options: nil) # rubocop:disable Metrics/ParameterLists
+        super()
+        @user = user
+        @host = host
+        @port = port
+        @password = password
+        @database = database
+        @extra_options = extra_options
+      end
+
+      def command
+        parts = []
+        parts << "pg_dump"
+        parts << extra_options if extra_options
+        parts << "--username=#{Shellwords.escape(@user)}" if @user
+        parts << "--host=#{@host}" if @host
+        parts << "--port=#{@port}" if @port
+        parts << @database
+        parts.join(" ")
+      end
+
+      def env
+        {
+          "PGPASSWORD" => @password,
+        }.compact
+      end
+    end
+  end
+end

data/lib/sg_tiny_backup/commands/tar.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+require "shellwords"
+require "open3"
+require_relative "base"
+
+module SgTinyBackup
+  module Commands
+    class Tar < Base
+      def initialize(paths: [], optional_paths: [])
+        super()
+        @paths = paths
+        @optional_paths = optional_paths
+      end
+
+      def command
+        if target_file_paths.empty?
+          # Create empty tar archive.
+          "tar -c -T /dev/null"
+        else
+          cmd = ["tar -c"]
+          target_file_paths.map do |path|
+            cmd << Shellwords.escape(path)
+          end
+          cmd.join(" ")
+        end
+      end
+
+      def success_codes
+        if self.class.gnu_tar?
+          # GNU tar's exit code 1 means that some files were changed while being archived.
+          # See https://www.gnu.org/software/tar/manual/html_section/Synopsis.html
+          [0, 1]
+        else
+          [0]
+        end
+      end
+
+      def warning_message
+        "tar: missing files: #{missing_optinal_file_paths.join(", ")}" unless missing_optinal_file_paths.empty?
+      end
+
+      private
+
+      def existing_optional_file_paths
+        @existing_optional_file_paths ||= @optional_paths.select { |path| File.file?(path) || File.directory?(path) }
+      end
+
+      def missing_optinal_file_paths
+        @missing_optinal_file_paths ||= @optional_paths - existing_optional_file_paths
+      end
+
+      def target_file_paths
+        @target_file_paths ||= @paths + existing_optional_file_paths
+      end
+
+      class << self
+        def gnu_tar?
+          unless defined?(@gnu_tar)
+            out, _err, status = Open3.capture3("tar --version")
+            @gnu_tar = status.success? && out.match?(/GNU/)
+          end
+          @gnu_tar
+        end
+      end
+    end
+  end
+end

data/lib/sg_tiny_backup/commands.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require "sg_tiny_backup/commands/aws_cli"
+require "sg_tiny_backup/commands/gzip"
+require "sg_tiny_backup/commands/mysql_dump"
+require "sg_tiny_backup/commands/openssl"
+require "sg_tiny_backup/commands/pg_dump"
+require "sg_tiny_backup/commands/tar"

data/lib/sg_tiny_backup/config.rb

@@ -0,0 +1,71 @@
+# frozen_string_literal: true
+
+module SgTinyBackup
+  class Config
+    KEY_PG_DUMP = "pg_dump"
+    KEY_MYSQLDUMP = "mysqldump"
+    KEY_S3 = "s3"
+    KEY_DB = "db"
+    KEY_ENCRYPTION_KEY = "encryption_key"
+    KEY_EXPECTED_UPLOAD_SIZE = "expected_upload_size"
+    KEY_LOG = "log"
+    KEY_FILES = "files"
+    KEY_OPTIONAL_FILES = "optional_files"
+    KEY_GZIP = "gzip"
+
+    attr_reader :s3, :encryption_key, :pg_dump, :mysqldump, :db, :gzip
+
+    def initialize(s3:, encryption_key:, pg_dump: nil, mysqldump: nil, db: nil, log: nil, gzip: nil) # rubocop:disable Metrics/ParameterLists
+      @s3 = s3
+      @encryption_key = encryption_key
+      @pg_dump = pg_dump || {}
+      @mysqldump = mysqldump
+      @db = db || self.class.rails_db_config
+      @log = log || {}
+      @gzip = gzip || {}
+    end
+
+    def log_file_paths
+      @log[KEY_FILES] || []
+    end
+
+    def optional_log_file_paths
+      @log[KEY_OPTIONAL_FILES] || []
+    end
+
+    class << self
+      def read(io)
+        yaml = Utils.load_yaml_with_erb(io)
+        Config.new(
+          s3: yaml[KEY_S3],
+          encryption_key: yaml[KEY_ENCRYPTION_KEY],
+          db: yaml[KEY_DB],
+          pg_dump: yaml[KEY_PG_DUMP],
+          mysqldump: yaml[KEY_MYSQLDUMP],
+          log: yaml[KEY_LOG],
+          gzip: yaml[KEY_GZIP]
+        )
+      end
+
+      def read_file(path)
+        File.open(path, "r") do |f|
+          read(f)
+        end
+      end
+
+      def generate_template_file(path)
+        template_path = File.expand_path("./templates/sg_tiny_backup.yml", __dir__)
+        FileUtils.cp(template_path, path)
+      end
+
+      def rails_db_config
+        return unless defined?(Rails)
+
+        db_config = File.open(Rails.root.join("config", "database.yml"), "r") do |f|
+          Utils.load_yaml_with_erb(f)
+        end
+        db_config[Rails.env]
+      end
+    end
+  end
+end

data/lib/sg_tiny_backup/error.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module SgTinyBackup
+  class Error < StandardError
+  end
+
+  class BackupFailed < Error
+  end
+
+  class BackupWarning < Error
+  end
+
+  class SpawnError < Error
+    def initialize(msg, inner_error)
+      inner_message = inner_error.message
+      inner_class_name = inner_error.class.name
+
+      message = msg.dup
+      message << "\n--- Inner error ---\n"
+      message << "#{inner_class_name}: " unless inner_message.start_with?(inner_class_name)
+      message << inner_message
+      super(message)
+    end
+  end
+end

data/lib/sg_tiny_backup/pipeline.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require "forwardable"
+require "sg_tiny_backup/spawner"
+
+module SgTinyBackup
+  class Pipeline
+    extend Forwardable
+
+    delegate stdout: :@spawner
+    delegate stderr: :@spawner
+    delegate succeeded?: :@spawner
+
+    def initialize(output_path: nil)
+      @commands = []
+      @spawner = build_spawner
+      @output_path = output_path
+    end
+
+    def <<(command)
+      @commands << command
+      self
+    end
+
+    def run
+      @spawner = build_spawner
+      @spawner.spawn_and_wait
+    end
+
+    def plain_commands
+      @commands.map(&:command)
+    end
+
+    def env
+      @commands.map(&:env).reduce({}, &:merge)
+    end
+
+    def failed?
+      !succeeded?
+    end
+
+    def error_message
+      [@spawner.stderr_message, @spawner.exit_code_error_message].join
+    end
+
+    def warning_message
+      messages = [commands_warning_message]
+      messages << @spawner.stderr_message if succeeded? && !@spawner.stderr_message.empty?
+      messages.reject { |str| str.nil? || str.empty? }.join("\n")
+    end
+
+    private
+
+    def commands_warning_message
+      @commands.filter_map(&:warning_message).join("\n")
+    end
+
+    def build_spawner
+      Spawner.new(commands: @commands, env: env, output_path: @output_path)
+    end
+  end
+end

data/lib/sg_tiny_backup/pipeline_builders/base.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require_relative "s3"
+
+module SgTinyBackup
+  module PipelineBuilders
+    class Base
+      attr_reader :s3_config
+
+      def initialize(config:, basename:, local:, s3_config:, extension:)
+        @config = config
+        @basename = basename
+        @local = local
+        @s3_config = s3_config
+        @extension = extension
+      end
+
+      def base_filename
+        "#{@basename}.#{@extension}"
+      end
+
+      def destination_url
+        S3.build_s3_url(s3_config: s3_config, base_filename: base_filename)
+      end
+
+      def local?
+        @local
+      end
+
+      private
+
+      def aws_cli_command
+        S3.build_aws_cli_command(s3_config: s3_config, destination_url: destination_url)
+      end
+    end
+  end
+end

data/lib/sg_tiny_backup/pipeline_builders/db.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require "sg_tiny_backup/commands"
+require "sg_tiny_backup/pipeline"
+require_relative "base"
+
+module SgTinyBackup
+  module PipelineBuilders
+    class Db < Base
+      def initialize(config:, basename:, local:)
+        super(
+          config: config,
+          basename: basename,
+          local: local,
+          s3_config: config.s3["db"],
+          extension: "sql.gz.enc"
+        )
+      end
+
+      def build
+        output_path = base_filename if local?
+        pl = Pipeline.new(output_path: output_path)
+        pl << db_dump_command
+        pl << Commands::Gzip.new(level: @config.gzip["level"])
+        pl << Commands::Openssl.new(password: @config.encryption_key)
+        pl << aws_cli_command unless local?
+        pl
+      end
+
+      private
+
+      def db_dump_command
+        db_config = @config.db
+        adapter = db_config["adapter"]
+        raise "database adapter is not specified in your config." if adapter.nil?
+
+        case adapter
+        when "postgresql"
+          pg_dump_command(db_config)
+        when "mysql2"
+          mysql_dump_command(db_config)
+        else
+          raise "database adapter `#{adapter}` is not supported."
+        end
+      end
+
+      def pg_dump_command(db_config)
+        Commands::PgDump.new(
+          database: db_config["database"],
+          host: db_config["host"],
+          port: db_config["port"],
+          user: db_config["username"] || db_config["user"],
+          password: db_config["password"],
+          extra_options: @config.pg_dump["extra_options"]
+        )
+      end
+
+      def mysql_dump_command(db_config)
+        Commands::MysqlDump.new(
+          database: db_config["database"],
+          host: db_config["host"],
+          port: db_config["port"],
+          user: db_config["username"] || db_config["user"],
+          password: db_config["password"],
+          extra_options: @config.mysqldump["extra_options"]
+        )
+      end
+    end
+  end
+end

data/lib/sg_tiny_backup/pipeline_builders/log.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require "sg_tiny_backup/commands"
+require "sg_tiny_backup/pipeline"
+require_relative "base"
+
+module SgTinyBackup
+  module PipelineBuilders
+    class Log < Base
+      def initialize(config:, basename:, local:)
+        super(
+          config: config,
+          basename: basename,
+          local: local,
+          s3_config: config.s3["log"],
+          extension: "tar.gz"
+        )
+      end
+
+      def build
+        output_path = base_filename if local?
+        pl = Pipeline.new(output_path: output_path)
+        pl << Commands::Tar.new(paths: @config.log_file_paths, optional_paths: @config.optional_log_file_paths)
+        pl << Commands::Gzip.new(level: @config.gzip["level"])
+        pl << aws_cli_command unless local?
+        pl
+      end
+
+      private
+
+      def extension
+        "tar.gz"
+      end
+    end
+  end
+end

data/lib/sg_tiny_backup/pipeline_builders/s3.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "sg_tiny_backup/commands/aws_cli"
+
+module SgTinyBackup
+  module PipelineBuilders
+    module S3
+      class << self
+        def build_aws_cli_command(s3_config:, destination_url:)
+          expected_size = s3_config["expected_upload_size"].to_i if s3_config["expected_upload_size"]
+          Commands::AwsCli.new(
+            destination_url: destination_url,
+            access_key_id: s3_config["access_key_id"],
+            secret_access_key: s3_config["secret_access_key"],
+            expected_size: expected_size
+          )
+        end
+
+        def build_s3_url(s3_config:, base_filename:)
+          object_path = [s3_config["prefix"], base_filename].join("_")
+          "s3://#{File.join(s3_config["bucket"], object_path)}"
+        end
+      end
+    end
+  end
+end

data/lib/sg_tiny_backup/railtie.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module SgTinyBackup
+  class Railtie < ::Rails::Railtie
+    rake_tasks do
+      load File.expand_path("../tasks/sg_tiny_backup.rake", __dir__)
+    end
+  end
+end

data/lib/sg_tiny_backup/runner.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "sg_tiny_backup/pipeline"
+require "sg_tiny_backup/error"
+require "sg_tiny_backup/pipeline_builders/db"
+require "sg_tiny_backup/pipeline_builders/log"
+
+module SgTinyBackup
+  class Runner
+    attr_reader :target
+
+    TARGET_DB = "db"
+    TARGET_LOG = "log"
+
+    def initialize(config:, basename:, target: "db", local: false, pipeline_builder: nil)
+      @config = config
+      @basename = basename
+      @target = target
+      @local = local
+      @pipeline_builder = pipeline_builder
+    end
+
+    def run # rubocop:disable Naming/PredicateMethod
+      pipeline.run
+      handle_warning(pipeline.warning_message)
+      handle_error(pipeline.error_message) if pipeline.failed?
+      pipeline.succeeded?
+    end
+
+    def plain_commands
+      pipeline.plain_commands
+    end
+
+    def piped_command
+      plain_commands.join(" | ")
+    end
+
+    def env
+      pipeline.env
+    end
+
+    def s3_destination_url
+      pipeline_builder.destination_url
+    end
+
+    def base_filename
+      pipeline_builder.base_filename
+    end
+
+    private
+
+    def pipeline
+      @pipeline ||= pipeline_builder.build
+    end
+
+    def pipeline_builder
+      @pipeline_builder ||=
+        case target
+        when TARGET_DB
+          SgTinyBackup::PipelineBuilders::Db.new(config: @config, basename: @basename, local: @local)
+        when TARGET_LOG
+          SgTinyBackup::PipelineBuilders::Log.new(config: @config, basename: @basename, local: @local)
+        end
+    end
+
+    def handle_warning(message)
+      return if message.empty?
+
+      SgTinyBackup.logger.warn message
+    end
+
+    def handle_error(message)
+      SgTinyBackup.logger.error message
+      raise BackupFailed, message if SgTinyBackup.raise_on_error
+    end
+  end
+end