backy_rb 0.1.3 → 0.1.4

data/lib/backy/pg_dump.rb

@@ -1,32 +1,86 @@
+require "fileutils"
+require "etc"
+
 module Backy
   class PgDump
     include Db
     include AppConfig
 
     DUMP_DIR = "db/dump"
-    DUMP_CMD_OPTS = "--no-acl --no-owner --no-subscriptions --no-publications --exclude-table=awsdms_ddl_audit"
+    DUMP_CMD_OPTS = "--no-acl --no-owner --no-subscriptions --no-publications"
 
     def call
       FileUtils.mkdir_p(DUMP_DIR)
+      Logger.log("Starting backy for #{database}")
+
+      if use_parallel?
+        Logger.log("Using multicore dump with pigz")
+        parallel_backup
+      else
+        Logger.log("Using single core dump")
+        plain_text_backup
+      end
+    end
+
+    private
 
-      dump_file = "#{DUMP_DIR}/#{database}_#{whoami}@#{hostname}_#{Time.zone.now.strftime("%Y%m%d_%H%M%S")}.sql.gz"
+    def plain_text_backup
+      timestamp = Time.now.strftime("%Y%m%d_%H%M%S")
+      dump_file = "#{DUMP_DIR}/#{database}_#{whoami}@#{hostname}_single_#{timestamp}.sql.gz"
 
       cmd = "(#{pg_password_env}pg_dump #{pg_credentials} #{database} #{DUMP_CMD_OPTS} | gzip -9 > #{dump_file}) 2>&1 >> #{log_file}"
 
       print "Saving to #{dump_file} ... "
 
       if system(cmd)
-        puts "done"
+        Logger.success("done")
       else
-        puts "error. See #{log_file}"
-
+        Logger.error("error. See #{log_file}")
         return
       end
 
       dump_file
     end
 
-    private
+    def parallel_backup
+      timestamp = Time.now.strftime("%Y%m%d_%H%M%S")
+      dump_dir = "#{DUMP_DIR}/#{database}_dump_parallel_#{timestamp}"
+      dump_file = "#{dump_dir}.tar.gz"
+
+      pg_dump_cmd = "pg_dump -Z0 -j #{Etc.nprocessors} -Fd #{database} -f #{dump_dir} #{pg_credentials} #{DUMP_CMD_OPTS}"
+      tar_cmd = "tar -cf - #{dump_dir} | pigz -p #{Etc.nprocessors} > #{dump_file}"
+      cleanup_cmd = "rm -rf #{dump_dir}"
+
+      Logger.log("Running pg_dump #{database}")
+      if system("#{pg_password_env}#{pg_dump_cmd} 2>&1 >> #{log_file}")
+        Logger.log("pg_dump completed successfully.")
+      else
+        Logger.error("pg_dump failed. See #{log_file} for details.")
+        return
+      end
+
+      # Execute tar command
+      Logger.log("Compressing #{dump_dir}")
+      if system(tar_cmd)
+        Logger.log("Compression completed successfully.")
+      else
+        Logger.error("Compression failed. See #{log_file} for details.")
+        return
+      end
+
+      # Execute cleanup command
+      Logger.log("Cleaning up #{dump_dir}")
+      if system(cleanup_cmd)
+        Logger.log("Cleanup completed successfully.")
+      else
+        Logger.error("Cleanup failed. See #{log_file} for details.")
+        return
+      end
+
+      Logger.success("Backup process completed. Output file: #{dump_file}")
+
+      dump_file # Return the name of the dump file
+    end
 
     def hostname
       @hostname ||= `hostname`.strip

data/lib/backy/pg_restore.rb

@@ -3,25 +3,91 @@ module Backy
     include Db
     include AppConfig
 
+    DUMP_DIR = "db/dump"
+
     def initialize(file_name:)
       @file_name = file_name
     end
 
     def call
-      terminate_connection_sql = "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '#{database}' AND pid <> pg_backend_pid();"
-      cmd = "(#{pg_password_env}psql -c \"#{terminate_connection_sql};\" #{pg_credentials} #{database}; #{pg_password_env}dropdb #{pg_credentials} #{database}; #{pg_password_env}createdb #{pg_credentials} #{database}; gunzip -c #{file_name} | #{pg_password_env}psql #{pg_credentials} -q -d #{database}) 2>&1 >> #{log_file}"
+      pigz_installed = system('which pigz > /dev/null 2>&1')
+      multicore = Etc.nprocessors > 1
+      use_multicore = ENV["BACKY_USE_PARALLEL"] == "true"
 
-      print "Restoring #{database} from #{file_name} ... "
-
-      if system(cmd)
-        puts "done"
+      if pigz_installed && multicore && use_multicore
+        log_message('Using parallel restore with pigz')
+        parallel_restore
       else
-        puts "error. See #{log_file}"
+        log_message("Pigz not installed or system is not multicore")
+        log_message('Using plain text restore')
+        plain_text_restore
       end
     end
 
     private
 
     attr_reader :file_name
+
+    def plain_text_restore
+      cmd = "(#{pg_password_env}psql -c \"#{terminate_connection_sql};\" #{pg_credentials} #{database}; #{pg_password_env}dropdb #{pg_credentials} #{database}; #{pg_password_env}createdb #{pg_credentials} #{database}; gunzip -c #{file_name} | #{pg_password_env}psql #{pg_credentials} -q -d #{database}) 2>&1 >> #{log_file}"
+
+      log_message("Restoring #{database} from #{file_name} ...")
+      if system(cmd)
+        log_message("Database restoration completed successfully.")
+      else
+        log_message("Database restoration failed. See #{log_file} for details.")
+      end
+    end
+
+    def parallel_restore
+      timestamp = Time.now.strftime("%Y%m%d_%H%M%S")
+      dump_dir = "#{DUMP_DIR}/#{database}_dump_#{timestamp}"
+      FileUtils.mkdir_p(dump_dir)
+
+      decompress_cmd = "pigz -p #{Etc.nprocessors} -dc #{file_name} | tar -C #{dump_dir} --strip-components 3 -xf -"
+      restore_cmd = "pg_restore -j #{Etc.nprocessors} -Fd -O -d #{database} #{dump_dir}"
+
+      # Terminate connections and drop/create database
+      terminate_and_recreate_db = "(#{pg_password_env}psql -c \"#{terminate_connection_sql};\" #{pg_credentials} #{database}; #{pg_password_env}dropdb #{pg_credentials} #{database}; #{pg_password_env}createdb #{pg_credentials} #{database}) 2>&1 >> #{log_file}"
+
+      log_message("Terminating connections to #{database}")
+      if system(terminate_and_recreate_db)
+        log_message("Database connections terminated and database recreated.")
+      else
+        log_message("Error during database termination and recreation. See #{log_file}")
+        return
+      end
+
+      # Decompress and restore
+      log_message("Decompressing #{file_name} into #{dump_dir} ...")
+      if system(decompress_cmd)
+        log_message("Decompression completed successfully.")
+
+        # Check the expected file
+        unless File.exist?("#{dump_dir}/toc.dat")
+          log_message("toc.dat not found in #{dump_dir}.")
+          return
+        end
+      else
+        log_message("Decompression failed. See #{log_file} for details.")
+        return
+      end
+
+      log_message("Restoring database from #{dump_dir} ...")
+      if system(restore_cmd)
+        log_message("Database restoration completed successfully.")
+      else
+        log_message("Database restoration failed. See #{log_file} for details.")
+        return
+      end
+
+      log_message("Cleanup: Removing #{dump_dir} ...")
+      FileUtils.rm_rf(dump_dir)
+      log_message("Cleanup completed.")
+    end
+
+    def terminate_connection_sql
+      "SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE pg_stat_activity.datname = '#{database}' AND pid <> pg_backend_pid();"
+    end
   end
 end

data/lib/backy/version.rb

@@ -1,3 +1,3 @@
 module Backy
-  VERSION = "0.1.3"
+  VERSION = "0.1.4"
 end

data/lib/{backy.rb → backy_rb.rb}

@@ -1,12 +1,15 @@
 # frozen_string_literal: true
 
+require "active_support"
 require "active_support/core_ext/integer/time"
 require "active_support/core_ext/object/blank"
 
+require_relative "backy/cli"
 require_relative "backy/configuration"
 require_relative "backy/app_config"
 require_relative "backy/db"
 require_relative "backy/list"
+require_relative "backy/logger"
 require_relative "backy/pg_dump"
 require_relative "backy/pg_restore"
 require_relative "backy/s3"
@@ -27,4 +30,16 @@ module Backy
   def self.configuration
     @configuration ||= Configuration.new
   end
+
+  def self.setup
+    Logger.log("Setting up Backy...")
+    configuration.load
+    setup_logging
+  end
+
+  def self.setup_logging
+    log_file = configuration.log_file
+    log_dir = File.dirname(log_file)
+    FileUtils.mkdir_p(log_dir) unless Dir.exist?(log_dir)
+  end
 end

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: backy_rb
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.1.4
 platform: ruby
 authors:
 - Alexey Kharchenko
 - Martin Ulleberg
 - Pål André Sundt
 autorequire:
-bindir: exe
+bindir: bin
 cert_chain: []
-date: 2023-06-16 00:00:00.000000000 Z
+date: 2023-12-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -124,6 +124,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.9.6
+- !ruby/object:Gem::Dependency
+  name: zlib
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: rails
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: faker
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.1'
 - !ruby/object:Gem::Dependency
   name: activerecord
   requirement: !ruby/object:Gem::Requirement
@@ -173,10 +215,12 @@ email:
 - akharchenko@gmail.com
 - martin.ulleberg@gmail.com
 - pal@rubynor.com
-executables: []
+executables:
+- backy
 extensions: []
 extra_rdoc_files: []
 files:
+- ".backyrc.example"
 - ".drone.yml"
 - ".idea/.gitignore"
 - ".rspec"
@@ -186,70 +230,17 @@ files:
 - Gemfile.lock
 - README.md
 - Rakefile
-- backy.gemspec
+- backy_rb.gemspec
+- bin/backy
 - bucky.png
 - config/database.example.yml
-- dummy/psql/.gitattributes
-- dummy/psql/.gitignore
-- dummy/psql/.ruby-version
 - dummy/psql/Gemfile
-- dummy/psql/Gemfile.lock
-- dummy/psql/README.md
-- dummy/psql/Rakefile
-- dummy/psql/app/assets/config/manifest.js
-- dummy/psql/app/assets/images/.keep
-- dummy/psql/app/assets/stylesheets/application.css
-- dummy/psql/app/controllers/application_controller.rb
-- dummy/psql/app/controllers/concerns/.keep
-- dummy/psql/app/controllers/posts_controller.rb
-- dummy/psql/app/helpers/application_helper.rb
-- dummy/psql/app/helpers/posts_helper.rb
-- dummy/psql/app/models/application_record.rb
-- dummy/psql/app/models/concerns/.keep
-- dummy/psql/app/models/post.rb
-- dummy/psql/app/views/layouts/application.html.erb
-- dummy/psql/app/views/posts/_form.html.erb
-- dummy/psql/app/views/posts/_post.html.erb
-- dummy/psql/app/views/posts/edit.html.erb
-- dummy/psql/app/views/posts/index.html.erb
-- dummy/psql/app/views/posts/new.html.erb
-- dummy/psql/app/views/posts/show.html.erb
-- dummy/psql/bin/bundle
-- dummy/psql/bin/rails
-- dummy/psql/bin/rake
-- dummy/psql/bin/setup
-- dummy/psql/config.ru
-- dummy/psql/config/application.rb
-- dummy/psql/config/boot.rb
-- dummy/psql/config/credentials.yml.enc
-- dummy/psql/config/database.yml
-- dummy/psql/config/environment.rb
-- dummy/psql/config/environments/development.rb
-- dummy/psql/config/environments/production.rb
-- dummy/psql/config/environments/test.rb
-- dummy/psql/config/locales/en.yml
-- dummy/psql/config/master.key
-- dummy/psql/config/puma.rb
-- dummy/psql/config/routes.rb
-- dummy/psql/db/migrate/20230330203226_create_posts.rb
-- dummy/psql/db/schema.rb
-- dummy/psql/db/seeds.rb
-- dummy/psql/log/.keep
-- dummy/psql/public/404.html
-- dummy/psql/public/422.html
-- dummy/psql/public/500.html
-- dummy/psql/public/apple-touch-icon-precomposed.png
-- dummy/psql/public/apple-touch-icon.png
-- dummy/psql/public/favicon.ico
-- dummy/psql/public/robots.txt
-- dummy/psql/tmp/.keep
-- dummy/psql/tmp/pids/.keep
-- lib/backy.rb
 - lib/backy/app_config.rb
 - lib/backy/cli.rb
 - lib/backy/configuration.rb
 - lib/backy/db.rb
 - lib/backy/list.rb
+- lib/backy/logger.rb
 - lib/backy/pg_dump.rb
 - lib/backy/pg_restore.rb
 - lib/backy/railtie.rb
@@ -258,6 +249,7 @@ files:
 - lib/backy/s3_load.rb
 - lib/backy/s3_save.rb
 - lib/backy/version.rb
+- lib/backy_rb.rb
 - lib/tasks/backy_tasks.rake
 homepage: https://rubynor.com
 licenses: []
@@ -280,7 +272,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.1
+rubygems_version: 3.4.22
 signing_key:
 specification_version: 4
 summary: Backy is a powerful and user-friendly database backup gem designed specifically

data/dummy/psql/.gitattributes

@@ -1,7 +0,0 @@
-# See https://git-scm.com/docs/gitattributes for more about git attribute files.
-
-# Mark the database schema as having been generated.
-db/schema.rb linguist-generated
-
-# Mark any vendored files as having been vendored.
-vendor/* linguist-vendored

data/dummy/psql/.gitignore

@@ -1,22 +0,0 @@
-# See https://help.github.com/articles/ignoring-files for more about ignoring files.
-#
-# If you find yourself ignoring temporary files generated by your text editor
-# or operating system, you probably want to add a global ignore instead:
-#   git config --global core.excludesfile '~/.gitignore_global'
-
-# Ignore bundler config.
-/.bundle
-
-# Ignore all logfiles and tempfiles.
-/log/*
-/tmp/*
-!/log/.keep
-!/tmp/.keep
-
-# Ignore pidfiles, but keep the directory.
-/tmp/pids/*
-!/tmp/pids/
-!/tmp/pids/.keep
-
-
-/public/assets

data/dummy/psql/.ruby-version

@@ -1 +0,0 @@
-ruby-3.2.0

data/dummy/psql/Gemfile.lock

@@ -1,207 +0,0 @@
-PATH
-  remote: ../..
-  specs:
-    backy (0.1.3)
-      activerecord (>= 4.0)
-      activesupport (>= 4.0)
-      aws-sdk-s3 (>= 1.117)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    actioncable (7.0.4.3)
-      actionpack (= 7.0.4.3)
-      activesupport (= 7.0.4.3)
-      nio4r (~> 2.0)
-      websocket-driver (>= 0.6.1)
-    actionmailbox (7.0.4.3)
-      actionpack (= 7.0.4.3)
-      activejob (= 7.0.4.3)
-      activerecord (= 7.0.4.3)
-      activestorage (= 7.0.4.3)
-      activesupport (= 7.0.4.3)
-      mail (>= 2.7.1)
-      net-imap
-      net-pop
-      net-smtp
-    actionmailer (7.0.4.3)
-      actionpack (= 7.0.4.3)
-      actionview (= 7.0.4.3)
-      activejob (= 7.0.4.3)
-      activesupport (= 7.0.4.3)
-      mail (~> 2.5, >= 2.5.4)
-      net-imap
-      net-pop
-      net-smtp
-      rails-dom-testing (~> 2.0)
-    actionpack (7.0.4.3)
-      actionview (= 7.0.4.3)
-      activesupport (= 7.0.4.3)
-      rack (~> 2.0, >= 2.2.0)
-      rack-test (>= 0.6.3)
-      rails-dom-testing (~> 2.0)
-      rails-html-sanitizer (~> 1.0, >= 1.2.0)
-    actiontext (7.0.4.3)
-      actionpack (= 7.0.4.3)
-      activerecord (= 7.0.4.3)
-      activestorage (= 7.0.4.3)
-      activesupport (= 7.0.4.3)
-      globalid (>= 0.6.0)
-      nokogiri (>= 1.8.5)
-    actionview (7.0.4.3)
-      activesupport (= 7.0.4.3)
-      builder (~> 3.1)
-      erubi (~> 1.4)
-      rails-dom-testing (~> 2.0)
-      rails-html-sanitizer (~> 1.1, >= 1.2.0)
-    activejob (7.0.4.3)
-      activesupport (= 7.0.4.3)
-      globalid (>= 0.3.6)
-    activemodel (7.0.4.3)
-      activesupport (= 7.0.4.3)
-    activerecord (7.0.4.3)
-      activemodel (= 7.0.4.3)
-      activesupport (= 7.0.4.3)
-    activestorage (7.0.4.3)
-      actionpack (= 7.0.4.3)
-      activejob (= 7.0.4.3)
-      activerecord (= 7.0.4.3)
-      activesupport (= 7.0.4.3)
-      marcel (~> 1.0)
-      mini_mime (>= 1.1.0)
-    activesupport (7.0.4.3)
-      concurrent-ruby (~> 1.0, >= 1.0.2)
-      i18n (>= 1.6, < 2)
-      minitest (>= 5.1)
-      tzinfo (~> 2.0)
-    aws-eventstream (1.2.0)
-    aws-partitions (1.779.0)
-    aws-sdk-core (3.174.0)
-      aws-eventstream (~> 1, >= 1.0.2)
-      aws-partitions (~> 1, >= 1.651.0)
-      aws-sigv4 (~> 1.5)
-      jmespath (~> 1, >= 1.6.1)
-    aws-sdk-kms (1.66.0)
-      aws-sdk-core (~> 3, >= 3.174.0)
-      aws-sigv4 (~> 1.1)
-    aws-sdk-s3 (1.124.0)
-      aws-sdk-core (~> 3, >= 3.174.0)
-      aws-sdk-kms (~> 1)
-      aws-sigv4 (~> 1.4)
-    aws-sigv4 (1.5.2)
-      aws-eventstream (~> 1, >= 1.0.2)
-    builder (3.2.4)
-    concurrent-ruby (1.2.2)
-    crass (1.0.6)
-    date (3.3.3)
-    debug (1.7.2)
-      irb (>= 1.5.0)
-      reline (>= 0.3.1)
-    erubi (1.12.0)
-    faker (3.1.1)
-      i18n (>= 1.8.11, < 2)
-    globalid (1.1.0)
-      activesupport (>= 5.0)
-    i18n (1.12.0)
-      concurrent-ruby (~> 1.0)
-    io-console (0.6.0)
-    irb (1.6.3)
-      reline (>= 0.3.0)
-    jmespath (1.6.2)
-    loofah (2.19.1)
-      crass (~> 1.0.2)
-      nokogiri (>= 1.5.9)
-    mail (2.8.1)
-      mini_mime (>= 0.1.1)
-      net-imap
-      net-pop
-      net-smtp
-    marcel (1.0.2)
-    method_source (1.0.0)
-    mini_mime (1.1.2)
-    minitest (5.18.0)
-    net-imap (0.3.4)
-      date
-      net-protocol
-    net-pop (0.1.2)
-      net-protocol
-    net-protocol (0.2.1)
-      timeout
-    net-smtp (0.3.3)
-      net-protocol
-    nio4r (2.5.8)
-    nokogiri (1.14.2-arm64-darwin)
-      racc (~> 1.4)
-    nokogiri (1.14.2-x86_64-linux)
-      racc (~> 1.4)
-    pg (1.4.6)
-    puma (5.6.5)
-      nio4r (~> 2.0)
-    racc (1.6.2)
-    rack (2.2.6.4)
-    rack-test (2.1.0)
-      rack (>= 1.3)
-    rails (7.0.4.3)
-      actioncable (= 7.0.4.3)
-      actionmailbox (= 7.0.4.3)
-      actionmailer (= 7.0.4.3)
-      actionpack (= 7.0.4.3)
-      actiontext (= 7.0.4.3)
-      actionview (= 7.0.4.3)
-      activejob (= 7.0.4.3)
-      activemodel (= 7.0.4.3)
-      activerecord (= 7.0.4.3)
-      activestorage (= 7.0.4.3)
-      activesupport (= 7.0.4.3)
-      bundler (>= 1.15.0)
-      railties (= 7.0.4.3)
-    rails-dom-testing (2.0.3)
-      activesupport (>= 4.2.0)
-      nokogiri (>= 1.6)
-    rails-html-sanitizer (1.5.0)
-      loofah (~> 2.19, >= 2.19.1)
-    railties (7.0.4.3)
-      actionpack (= 7.0.4.3)
-      activesupport (= 7.0.4.3)
-      method_source
-      rake (>= 12.2)
-      thor (~> 1.0)
-      zeitwerk (~> 2.5)
-    rake (13.0.6)
-    reline (0.3.3)
-      io-console (~> 0.5)
-    sprockets (4.2.0)
-      concurrent-ruby (~> 1.0)
-      rack (>= 2.2.4, < 4)
-    sprockets-rails (3.4.2)
-      actionpack (>= 5.2)
-      activesupport (>= 5.2)
-      sprockets (>= 3.0.0)
-    thor (1.2.1)
-    timeout (0.3.2)
-    tzinfo (2.0.6)
-      concurrent-ruby (~> 1.0)
-    websocket-driver (0.7.5)
-      websocket-extensions (>= 0.1.0)
-    websocket-extensions (0.1.5)
-    zeitwerk (2.6.7)
-
-PLATFORMS
-  arm64-darwin-22
-  x86_64-linux
-
-DEPENDENCIES
-  backy!
-  debug
-  faker (~> 3.1)
-  pg (~> 1.1)
-  puma (~> 5.0)
-  rails (~> 7.0.4, >= 7.0.4.2)
-  sprockets-rails
-  tzinfo-data
-
-RUBY VERSION
-   ruby 3.2.0p0
-
-BUNDLED WITH
-   2.4.8

data/dummy/psql/README.md

@@ -1,24 +0,0 @@
-# README
-
-This README would normally document whatever steps are necessary to get the
-application up and running.
-
-Things you may want to cover:
-
-* Ruby version
-
-* System dependencies
-
-* Configuration
-
-* Database creation
-
-* Database initialization
-
-* How to run the test suite
-
-* Services (job queues, cache servers, search engines, etc.)
-
-* Deployment instructions
-
-* ...

data/dummy/psql/Rakefile

@@ -1,6 +0,0 @@
-# Add your own tasks in files placed in lib/tasks ending in .rake,
-# for example lib/tasks/capistrano.rake, and they will automatically be available to Rake.
-
-require_relative "config/application"
-
-Rails.application.load_tasks

data/dummy/psql/app/assets/config/manifest.js

@@ -1,2 +0,0 @@
-//= link_tree ../images
-//= link_directory ../stylesheets .css