backy_rb 0.1.8 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9d66c6c91f2f5538af37756b5dc6e2759330b0b33914647cf0aa0b1f3ee376cc
-  data.tar.gz: 20856cc301a8590f8d60200664995b0ab408f76f15e077bc931d3c828fc11658
+  metadata.gz: b7bb1c2b1110d5fe15106b8bb78e8e07201e2318156aa6f67a4349eb92720a06
+  data.tar.gz: 5a8ae6f47b04ac3480cecedf77987166d5d630cc511496a734cddf5f7659e871
 SHA512:
-  metadata.gz: 62cd5a81471ef89e5ff57195b32ae4e9307a3f31a14f3bc9ee1a931b6534c4922406198aa30756fda197f2c60c279fefd76b46912419d348f3fe8922bccca978
-  data.tar.gz: 1541af37d97d53e039b598da4ce53dd1a119e69f77fed1d9251ea6f839a31a72e3d06a324ed3832f053670d0c41350e013b40418ad425971ee73f31df1d95e76
+  metadata.gz: 956ae711507992a7c75a3340d108b2df9e8da541ce9e90d346b1c0249bd22a414a6bf41f5e78f901f8b81ec68fd626a6f4cafbc8833d02a6eaea5f55563a0796
+  data.tar.gz: ca0b9c66ff07d33bd204df31e400690e53ca3a7108d8267eed734e9f5bc3d280046d1033856fbf13f89c297c38821d212ae573f4ea1d30b49eda95eb135bb624

data/.backyrc.example

@@ -1,17 +1,33 @@
 # ~/.backyrc
-defaults:
+shared:
   use_parallel: true
-  s3:
-    access_key_id: "YOUR_AWS_ACCESS_KEY_ID"
-    secret_access_key: "YOUR_AWS_SECRET_ACCESS_KEY"
-    region: "eu-central-1"
-    bucket: "your-s3-bucket-name"
-  database:
-    adapter: "postgresql"
-    host: "localhost"
-    port: 5432
-    username: "your-db-username"
-    password: "your-db-password"
-    database_name: "your-database-name"
-  local:
-    backup_path: "/path/to/your/local/backup/directory"
+  pause_replication: true
+  s3_access_key: "YOUR_AWS_ACCESS_KEY_ID"
+  s3_secret: "YOUR_AWS_SECRET_ACCESS_KEY"
+  s3_region: "eu-central-1"
+  s3_bucket: "your-s3-bucket-name"
+  s3_prefix: "./db/dump/"
+  pg_host: "localhost"
+  pg_port: 5432
+  pg_username: "your-db-username"
+  pg_password: "your-db-password"
+  pg_database: "your-database-name"
+  app_name: "backy"
+  environment: "development"
+  log_file: "./log/backy.log"
+  local_backup_path: "/path/to/your/local/backup/directory"
+
+production:
+  pg_host: "production-host"
+  s3_bucket: "production-s3-bucket-name"
+  log_file: "./log/production_backy.log"
+
+staging:
+  pg_host: "staging-host"
+  s3_bucket: "staging-s3-bucket-name"
+  log_file: "./log/staging_backy.log"
+
+development:
+  pg_host: "localhost"
+  s3_bucket: "development-s3-bucket-name"
+  log_file: "./log/development_backy.log"

data/CHANGELOG.md

@@ -3,6 +3,18 @@
 All notable changes to `Backy` will be documented in this file.
 
 ## [Unreleased]
+## [0.2.1] - 2024-06-24
+### Fixed
+- Fix bug where `push` is not working because of missing filename
+
+## [0.2.0] - 2024-06-24
+### Added
+- Support for turning off replication
+- Added support for config by environment in .backyrc
+
+### Changed
+- Breaking change; Change in config keys
+- Internal refactoring
 
 ## [0.1.8] - 2024-06-22
 ### Fixed

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    backy_rb (0.1.8)
+    backy_rb (0.2.1)
       activerecord (>= 4.0)
       activesupport (>= 4.0)
       aws-sdk-s3 (>= 1.117)

data/lib/backy/app_config.rb

@@ -1,4 +1,4 @@
-require 'forwardable'
+require "forwardable"
 
 module Backy
   module AppConfig

data/lib/backy/configuration.rb

@@ -1,29 +1,44 @@
+require "fileutils"
+require "etc"
+require "open3"
+require "yaml"
+require "uri"
+
 module Backy
   class Configuration
-    attr_writer(
-      :pg_host,
-      :pg_port,
-      :pg_database,
-      :pg_username,
-      :pg_password,
-      :s3_region,
-      :s3_access_key,
-      :s3_secret,
-      :s3_bucket,
-      :s3_prefix,
-      :app_name,
-      :environment,
-      :use_parallel,
-      :log_file
-    )
+    DEFAULTS = {
+      pg_host: nil,
+      pg_port: nil,
+      pg_database: nil,
+      pg_username: nil,
+      pg_password: nil,
+      s3_region: nil,
+      s3_access_key: nil,
+      s3_secret: nil,
+      s3_bucket: nil,
+      s3_prefix: "./db/dump/",
+      app_name: "backy",
+      environment: "development",
+      use_parallel: false,
+      pause_replication: true,
+      log_file: "./log/backy.log",
+      local_backup_path: nil
+    }.freeze
+
+    CONFIG_FILE_NAME = ".backyrc"
+
+    attr_accessor(*DEFAULTS.keys)
+
+    def initialize
+      DEFAULTS.each do |key, value|
+        instance_variable_set("@#{key}", value)
+      end
+    end
 
     def load
-      local_config_file = File.join(Dir.pwd, '.backyrc')
-      global_config_file = File.join(Dir.home, '.backyrc')
+      load_config_file
 
-      config_file = File.exist?(local_config_file) ? local_config_file : global_config_file
-      Logger.log("Loading configuration from #{config_file}...") if File.exist?(config_file)
-      load_from_file(config_file) if File.exist?(config_file)
+      load_from_env
     end
 
     def pg_url=(url)
@@ -40,56 +55,14 @@ module Backy
       @pg_url ||= ENV["PG_URL"]
     end
 
-    def pg_host
-      @pg_host ||= ENV["PG_HOST"]
-    end
-
-    def pg_port
-      @pg_port ||= ENV["PG_PORT"]
-    end
-
-    def pg_database
-      @pg_database ||= ENV["PG_DATABASE"]
-    end
-
-    def pg_username
-      @pg_username ||= ENV["PG_USERNAME"]
-    end
-
-    def pg_password
-      @pg_password ||= ENV["PG_PASSWORD"]
-    end
-
-    def s3_region
-      @s3_region ||= ENV["S3_REGION"]
-    end
-
-    def s3_access_key
-      @s3_access_key ||= ENV["S3_ACCESS_KEY"]
-    end
-
-    def s3_secret
-      @s3_secret ||= ENV["S3_SECRET"]
-    end
-
-    def s3_bucket
-      @s3_bucket ||= ENV["S3_BUCKET"]
-    end
-
-    def s3_prefix
-      @s3_prefix ||= ENV["S3_PREFIX"].presence || "/db/dump/"
-    end
-
-
-
-    def use_parallel
-      @use_parallel ||= ENV["BACKY_USE_PARALLEL"] == "true"
-    end
-
     def use_parallel?
       use_parallel && pigz_installed && multicore
     end
 
+    def pause_replication?
+      pause_replication
+    end
+
     # Detect if pigz binary is available
     # If it is, use it to speed up the dump
     # pigz is a parallel gzip implementation
@@ -103,14 +76,6 @@ module Backy
       @multicore ||= Etc.nprocessors > 1
     end
 
-    def app_name
-      @app_name ||= ENV["APP_NAME"].presence || "backy"
-    end
-
-    def environment
-      @environment ||= "development"
-    end
-
     def log_file
       @log_file ||= default_log_file
     end
@@ -120,37 +85,62 @@ module Backy
     def default_log_file
       if Gem.win_platform?
         # Windows default path
-        File.join(Dir.home, "AppData", "Local", "#{app_name}", "log", "#{app_name}.log")
+        File.join(Dir.home, "AppData", "Local", app_name, "log", app_name, ".log")
       else
         # Unix-like systems default path
-        File.join(Dir.home, ".local", "share", "#{app_name}", "log", "#{app_name}.log")
+        File.join(Dir.home, ".local", "share", app_name, "log", app_name, ".log")
+      end
+    end
+
+    def load_config_file
+      local_config_file = File.join(Dir.pwd, CONFIG_FILE_NAME)
+      global_config_file = File.join(Dir.home, CONFIG_FILE_NAME)
+
+      config_file = File.exist?(local_config_file) ? local_config_file : global_config_file
+      if File.exist?(config_file)
+        Logger.log("Loading configuration from #{config_file}...")
+        load_from_file(config_file)
       end
     end
 
     def load_from_file(file_path)
       configuration = YAML.load_file(file_path)
 
-      @s3_access_key = configuration.dig("defaults", "s3", "access_key_id")
-      @s3_secret = configuration.dig("defaults", "s3", "secret_access_key")
-      @s3_region = configuration.dig("defaults", "s3", "region")
-      @s3_bucket = configuration.dig("defaults", "s3", "bucket")
-      @s3_prefix = configuration.dig("defaults", "s3", "prefix") || s3_prefix
+      shared_config = configuration.fetch("shared", {})
+      environment_config = configuration.fetch(environment, {})
 
-      @pg_url = configuration.dig("defaults", "database", "pg_url")
-      if @pg_url
-        self.pg_url = @pg_url
-      else
-        @pg_host = configuration.dig("defaults", "database", "host")
-        @pg_port = configuration.dig("defaults", "database", "port")
-        @pg_username = configuration.dig("defaults", "database", "username")
-        @pg_password = configuration.dig("defaults", "database", "password")
-        @pg_database = configuration.dig("defaults", "database", "database_name")
+      merged_config = deep_merge(shared_config, environment_config)
+
+      apply_config(merged_config)
+    end
+
+    def apply_config(config)
+      config.each do |key, value|
+        instance_variable_set("@#{key}", value) if respond_to?("#{key}=")
       end
 
-      @app_name = configuration.dig("defaults", "app_name") || "backy"
-      @environment = configuration.dig("defaults", "environment") || "development"
-      @log_file = configuration.dig("defaults", "log", "file") || default_log_file
-      @use_parallel = configuration.dig("defaults", "use_parallel") || false
+      self.pg_url = @pg_url if @pg_url
+    end
+
+    def load_from_env
+      ENV.each do |key, value|
+        case key
+        when "PG_HOST" then @pg_host = value
+        when "PG_PORT" then @pg_port = value
+        when "PG_DATABASE" then @pg_database = value
+        when "PG_USERNAME" then @pg_username = value
+        when "PG_PASSWORD" then @pg_password = value
+        when "S3_REGION" then @s3_region = value
+        when "S3_ACCESS_KEY" then @s3_access_key = value
+        when "S3_SECRET" then @s3_secret = value
+        when "S3_BUCKET" then @s3_bucket = value
+        when "S3_PREFIX" then @s3_prefix = value
+        when "APP_NAME" then @app_name = value
+        when "BACKY_USE_PARALLEL" then @use_parallel = value == "true"
+        when "BACKY_PAUSE_REPLICATION" then @pause_replication = value == "true"
+        when "LOCAL_BACKUP_PATH" then @local_backup_path = value
+        end
+      end
     end
 
     def parse_postgres_uri(uri)
@@ -165,5 +155,17 @@ module Backy
         database_name: parsed_uri.path[1..]
       }
     end
+
+    def deep_merge(hash1, hash2)
+      merged = hash1.dup
+      hash2.each do |key, value|
+        merged[key] = if value.is_a?(Hash) && hash1[key].is_a?(Hash)
+          deep_merge(hash1[key], value)
+        else
+          value
+        end
+      end
+      merged
+    end
   end
 end

data/lib/backy/db.rb

@@ -1,4 +1,4 @@
-require 'forwardable'
+require "forwardable"
 
 module Backy
   module Db
@@ -12,6 +12,7 @@ module Backy
     def_delegator "Backy.configuration", :pg_username, :username
     def_delegator "Backy.configuration", :pg_password, :password
     def_delegator "Backy.configuration", :use_parallel?, :use_parallel?
+    def_delegator "Backy.configuration", :pause_replication?, :pause_replication?
 
     def pg_password_env
       password.present? ? "PGPASSWORD='#{password}' " : ""

data/lib/backy/logger.rb

@@ -1,11 +1,13 @@
-require 'thor'
+require "thor"
 
 module Backy
   class Logger
+    @log_messages = []
+
     # Logs a message with the specified color using Thor's shell
     def self.log(message, color = nil)
-      thor_shell = Thor::Base.shell.new
-      thor_shell.say("[#{Time.now.strftime("%Y-%m-%d %H:%M:%S")}] #{message}", color)
+      @log_messages << message
+      say("[#{Time.now.strftime("%Y-%m-%d %H:%M:%S")}] #{message}\n", color)
     end
 
     def self.success(message)
@@ -23,5 +25,17 @@ module Backy
     def self.error(message)
       log(message, :red)
     end
+
+    def self.say(message, color = nil)
+      thor_shell.say(message, color)
+    end
+
+    def self.log_messages
+      @log_messages
+    end
+
+    private_class_method def self.thor_shell
+      @thor_shell ||= Thor::Base.shell.new
+    end
   end
 end

data/lib/backy/pg_dump.rb

@@ -1,5 +1,6 @@
 require "fileutils"
 require "etc"
+require "open3"
 
 module Backy
   class PgDump
@@ -10,9 +11,46 @@ module Backy
     DUMP_CMD_OPTS = "--no-acl --no-owner --no-subscriptions --no-publications"
 
     def call
+      setup_backup_directory
+      log_start
+
+      dump_file = nil
+
+      begin
+        handle_replication { dump_file = backup }
+      rescue => e
+        Logger.error("An error occurred during backup: #{e.message}")
+      ensure
+        log_replication_resume if replica? && pause_replication?
+      end
+
+      dump_file
+    end
+
+    private
+
+    def setup_backup_directory
       FileUtils.mkdir_p(DUMP_DIR)
+    end
+
+    def log_start
       Logger.log("Starting backy for #{database}")
+    end
 
+    def handle_replication
+      if replica? && pause_replication?
+        if pause_replication
+          Logger.log("Replication paused.")
+          yield
+        else
+          Logger.error("Failed to pause replication. Aborting backup.")
+        end
+      else
+        yield
+      end
+    end
+
+    def backup
       if use_parallel?
         Logger.log("Using multicore dump with pigz")
         parallel_backup
@@ -22,64 +60,55 @@ module Backy
       end
     end
 
-    private
+    def log_replication_resume
+      if resume_replication
+        Logger.log("Replication resumed.")
+      else
+        Logger.error("Failed to resume replication. Manual intervention required.")
+      end
+    end
 
     def plain_text_backup
-      timestamp = Time.now.strftime("%Y%m%d_%H%M%S")
-      dump_file = "#{DUMP_DIR}/#{database}_#{whoami}@#{hostname}_single_#{timestamp}.sql.gz"
+      timestamp = current_timestamp
+      dump_file = "#{DUMP_DIR}/#{database}_#{whoami}@#{hostname}_#{timestamp}.sql.gz"
 
       cmd = "(#{pg_password_env}pg_dump #{pg_credentials} #{database} #{DUMP_CMD_OPTS} | gzip -9 > #{dump_file}) 2>&1 >> #{log_file}"
 
-      print "Saving to #{dump_file} ... "
+      Logger.log("Saving to #{dump_file} ... ")
 
-      if system(cmd)
-        Logger.success("done")
-      else
-        Logger.error("error. See #{log_file}")
-        return
-      end
+      execute_command(cmd, "error. See #{log_file}")
 
       dump_file
     end
 
     def parallel_backup
-      timestamp = Time.now.strftime("%Y%m%d_%H%M%S")
+      timestamp = current_timestamp
       dump_dir = "#{DUMP_DIR}/#{database}_dump_parallel_#{timestamp}"
       dump_file = "#{dump_dir}.tar.gz"
 
-      pg_dump_cmd = "pg_dump -Z0 -j #{Etc.nprocessors} -Fd #{database} -f #{dump_dir} #{pg_credentials} #{DUMP_CMD_OPTS}"
+      pg_dump_cmd = "#{pg_password_env}pg_dump -Z0 -j #{Etc.nprocessors} -Fd #{database} -f #{dump_dir} #{pg_credentials} #{DUMP_CMD_OPTS}"
       tar_cmd = "tar -cf - #{dump_dir} | pigz -p #{Etc.nprocessors} > #{dump_file}"
       cleanup_cmd = "rm -rf #{dump_dir}"
 
-      Logger.log("Running pg_dump #{database}")
-      if system("#{pg_password_env}#{pg_dump_cmd} 2>&1 >> #{log_file}")
-        Logger.log("pg_dump completed successfully.")
-      else
-        Logger.error("pg_dump failed. See #{log_file} for details.")
-        return
-      end
+      execute_command("#{pg_password_env}#{pg_dump_cmd} 2>&1 >> #{log_file}", "pg_dump failed. See #{log_file} for details.")
+      execute_command(tar_cmd, "Compression failed. See #{log_file} for details.")
+      execute_command(cleanup_cmd, "Cleanup failed. See #{log_file} for details.")
 
-      # Execute tar command
-      Logger.log("Compressing #{dump_dir}")
-      if system(tar_cmd)
-        Logger.log("Compression completed successfully.")
-      else
-        Logger.error("Compression failed. See #{log_file} for details.")
-        return
-      end
+      Logger.success("Backup process completed. Output file: #{dump_file}")
+
+      dump_file
+    end
 
-      # Execute cleanup command
-      Logger.log("Cleaning up #{dump_dir}")
-      if system(cleanup_cmd)
-        Logger.log("Cleanup completed successfully.")
+    def execute_command(cmd, error_message)
+      if system(cmd)
+        Logger.success("done")
       else
-        Logger.error("Cleanup failed. See #{log_file} for details.")
-        return
+        Logger.error(error_message)
       end
+    end
 
-      Logger.success("Backup process completed. Output file: #{dump_file}")
-
-      dump_file # Return the name of the dump file
+    def current_timestamp
+      Time.now.strftime("%Y%m%d_%H%M%S")
     end
 
     def hostname
@@ -89,5 +118,45 @@ module Backy
     def whoami
       @whoami ||= `whoami`.strip
     end
+
+    def pause_replication
+      query = "SELECT pg_wal_replay_pause();"
+      success, _output = execute_sql(query)
+      success
+    end
+
+    def resume_replication
+      query = "SELECT pg_wal_replay_resume();"
+      success, _output = execute_sql(query)
+      success
+    end
+
+    def execute_sql(query)
+      command = %(#{pg_password_env}psql #{pg_credentials} -d #{database} -c "#{query}")
+      output = ""
+      Open3.popen3(command) do |_stdin, stdout, stderr, wait_thr|
+        while (line = stdout.gets)
+          output << line
+        end
+        while (line = stderr.gets)
+          puts "Error: #{line}"
+        end
+        exit_status = wait_thr.value
+        [exit_status.success?, output]
+      end
+    end
+
+    def replica?
+      @is_replica ||= begin
+        query = "SELECT pg_is_in_recovery();"
+        success, output = execute_sql(query)
+        if success && output.include?("t")
+          Logger.log("Database is a replica.")
+          true
+        else
+          false
+        end
+      end
+    end
   end
 end

data/lib/backy/pg_restore.rb

@@ -10,16 +10,16 @@ module Backy
     end
 
     def call
-      pigz_installed = system('which pigz > /dev/null 2>&1')
+      pigz_installed = system("which pigz > /dev/null 2>&1")
       multicore = Etc.nprocessors > 1
       use_multicore = ENV["BACKY_USE_PARALLEL"] == "true"
 
       if pigz_installed && multicore && use_multicore
-        Logger.log('Using parallel restore with pigz')
+        Logger.log("Using parallel restore with pigz")
         parallel_restore
       else
         Logger.log("Pigz not installed or system is not multicore")
-        Logger.log('Using plain text restore')
+        Logger.log("Using plain text restore")
         plain_text_restore
       end
     end

data/lib/backy/s3.rb

@@ -1,5 +1,5 @@
 require "aws-sdk-s3"
-require 'forwardable'
+require "forwardable"
 
 module Backy
   module S3

data/lib/backy/version.rb

@@ -1,3 +1,3 @@
 module Backy
-  VERSION = "0.1.8"
+  VERSION = "0.2.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: backy_rb
 version: !ruby/object:Gem::Version
-  version: 0.1.8
+  version: 0.2.1
 platform: ruby
 authors:
 - Alexey Kharchenko
@@ -10,7 +10,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-06-22 00:00:00.000000000 Z
+date: 2024-06-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -301,7 +301,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.5
+rubygems_version: 3.4.22
 signing_key:
 specification_version: 4
 summary: Backy is a powerful and user-friendly database backup gem designed specifically