remote_database_importer 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: cc2634f7ebf5e2ab65295a75ce3a851bfdc12632be061376b921e20689a7e614
+  data.tar.gz: 62e21889fbbc2ecff1ec1d9eae582a112bb8121509c31118558bab4ed947c190
+SHA512:
+  metadata.gz: a2dab0f7513fba0965b73effd56c2aeeb96c24f85ace4fff2d97e40b0f13c960d517321e950381923c0534b565891c885c10e7ed36ebc0c51899cf0303c8bb4b
+  data.tar.gz: ff8507cff4411cb07295a3faddb54df8d263573c15baff4ffcfb493f5379956cc969fb08ca3edf1fc8790ed26618539134e99323a2afa48d8ca9290bf477dc22

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.standard.yml

@@ -0,0 +1,3 @@
+# For available configuration options, see:
+#   https://github.com/testdouble/standard
+ruby_version: 2.6

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+# Changelog
+
+## [0.1.0] - 2022-10-24
+
+- Initial release

data/Gemfile

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+source "https://rubygems.org"
+
+# Specify your gem's dependencies in remote_database_importer.gemspec
+gemspec
+
+gem "rake", "~> 13.0"
+
+gem "rspec", "~> 3.0"
+
+gem "standard", "~> 1.3"

data/Gemfile.lock

@@ -0,0 +1,80 @@
+PATH
+  remote: .
+  specs:
+    remote_database_importer (0.1.0)
+      colorize (~> 0.8)
+      thor (~> 1.2)
+      tty-config (~> 0.6)
+      tty-spinner (~> 0.9)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ast (2.4.2)
+    coderay (1.1.3)
+    colorize (0.8.1)
+    diff-lcs (1.5.0)
+    json (2.6.2)
+    method_source (1.0.0)
+    parallel (1.22.1)
+    parser (3.1.2.1)
+      ast (~> 2.4.1)
+    pry (0.14.1)
+      coderay (~> 1.1)
+      method_source (~> 1.0)
+    rainbow (3.1.1)
+    rake (13.0.6)
+    regexp_parser (2.6.0)
+    rexml (3.2.5)
+    rspec (3.11.0)
+      rspec-core (~> 3.11.0)
+      rspec-expectations (~> 3.11.0)
+      rspec-mocks (~> 3.11.0)
+    rspec-core (3.11.0)
+      rspec-support (~> 3.11.0)
+    rspec-expectations (3.11.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.11.0)
+    rspec-mocks (3.11.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.11.0)
+    rspec-support (3.11.1)
+    rubocop (1.35.1)
+      json (~> 2.3)
+      parallel (~> 1.10)
+      parser (>= 3.1.2.1)
+      rainbow (>= 2.2.2, < 4.0)
+      regexp_parser (>= 1.8, < 3.0)
+      rexml (>= 3.2.5, < 4.0)
+      rubocop-ast (>= 1.20.1, < 2.0)
+      ruby-progressbar (~> 1.7)
+      unicode-display_width (>= 1.4.0, < 3.0)
+    rubocop-ast (1.23.0)
+      parser (>= 3.1.1.0)
+    rubocop-performance (1.14.3)
+      rubocop (>= 1.7.0, < 2.0)
+      rubocop-ast (>= 0.4.0)
+    ruby-progressbar (1.11.0)
+    standard (1.16.1)
+      rubocop (= 1.35.1)
+      rubocop-performance (= 1.14.3)
+    thor (1.2.1)
+    tty-config (0.6.0)
+    tty-cursor (0.7.1)
+    tty-spinner (0.9.3)
+      tty-cursor (~> 0.7)
+    unicode-display_width (2.3.0)
+
+PLATFORMS
+  arm64-darwin-21
+  x86_64-linux
+
+DEPENDENCIES
+  pry (~> 0.14)
+  rake (~> 13.0)
+  remote_database_importer!
+  rspec (~> 3.0)
+  standard (~> 1.3)
+
+BUNDLED WITH
+   2.3.7

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2022 Leon
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,67 @@
+# RemoteDatabaseImporter
+**RemoteDatabaseImporter** is a small gem with one specific reason to exist: Dump remote databases and import it locally.
+
+**This gem is currently in beta phase!**  
+Its well possible that unexpected errors can occur.
+
+## Features
+- Define multiple environments (such as staging, production etc.)
+- Rails intergration via rake task
+- Decide for yourself if the dump should be done over ssh or if pg_dump should connect to the DB port directly
+- It can therefore be used for almost all hosting providers (Heroku, Kubernetes, self-hosted, etc.)
+
+## Installation
+
+Add this line to your application's gemfile:
+
+```ruby
+gem 'remote_database_importer'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install remote_database_importer
+
+## Usage
+Whenever you want current live data, you can run the command:
+
+```ruby
+rake remote_database:import
+```
+
+![Import Job Demo](readme_images/import-job.gif)
+
+### Config
+The settings for the different environments is stored in the `remote_database_importer.yml` file.  
+When you first run the rake task, it will dynamically create this file for you.
+
+
+![asdf](readme_images/config_sample.png)
+
+### DB Access
+The easiest and fastest way is to exchange your ssh-key with the server beforehand, so you don't have to enter a password.  
+Otherwise during the rake task execution a password entry is required.
+
+The effective dump call is as follows:
+```ruby
+"ssh SSH_USER@HOST -p SSH_PORT 'pg_dump -Fc -U DB_USER -d DB_NAME -h localhost -C' > DB_DUMP_LOCATION"
+or
+"pg_dump -Fc 'host=HOST dbname=DB_NAME user=DB_USER port=POSTGRES_PORT' > DB_DUMP_LOCATION"
+```
+
+## Limitations
+- At the moment only Postgres databases are supported
+- It has to run inside a Rails app. There is a CLI command `remote_database_importer import` as an alternative to the rake task, but there are still some Rails commands like `rails db:drop db:create - rails db:migrate`, which makes it currently not possible to use the gem outside of Rails
+- Not suitable for very large databases, you could run into SSH timeouts
+
+## Contributing
+
+Bug reports and pull requests are very welcome!
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+require "standard/rake"
+
+task default: %i[spec standard]

data/bin/remote_database_importer

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require 'thor'
+require_relative '../lib/remote_database_importer/operation'
+
+class RemoteDatabaseImporterCLI < Thor
+  desc "Restore remote database and import it locally", "asdf"
+  def import
+    importer = RemoteDatabaseImporter::Operation.new
+    importer.import
+  end
+end
+
+RemoteDatabaseImporterCLI.start

data/lib/railtie.rb

@@ -0,0 +1,10 @@
+require "remote_database_importer"
+require "rails"
+
+module RemoteDatabaseImporter
+  class Railtie < Rails::Railtie
+    rake_tasks do
+      load "tasks/remote_database_importer.rake"
+    end
+  end
+end

data/lib/remote_database_importer/config.rb

@@ -0,0 +1,99 @@
+module RemoteDatabaseImporter
+  class Config
+    require "tty/config"
+    require "colorize"
+
+    def initialize
+      @config = TTY::Config.new
+      @config.filename = "remote_database_importer"
+      @config.extname = ".yml"
+      @config.append_path Dir.pwd
+    end
+
+    def read_or_create_configfile
+      unless @config.exist?
+        puts "===========================================================".colorize(:green)
+        puts "Hi there! There is no config file yet, lets create one! 😄"
+        create_default_config
+        puts "===========================================================".colorize(:green)
+      end
+      @config.read
+    end
+
+    def ask(question, default: nil, options: nil)
+      question += " (#{options.join(" / ")})" if options.present?
+      question += " [#{default}]" if default.present?
+
+      puts question.colorize(:light_blue)
+      answer = $stdin.gets.chomp
+      answer.present? ? answer : default
+    end
+
+    def create_default_config
+      enter_new_environments = true
+      environment_count = 1
+
+      local_db_name = ask("Whats the name of the local database you wanna import to?", default: "myawesomeapp_development")
+      @config.set(:local_db_name, value: local_db_name)
+      puts
+
+      while enter_new_environments
+        puts "#{environment_count}. Environment".colorize(:green)
+        env = ask("Whats the name of the #{environment_count}. environment you wanna add?", default: "staging")
+        puts
+
+        puts "Database settings:".colorize(:green)
+        db_name = ask("Enter the DB name for the #{env} environment:", default: "myawesomeapp_#{env}")
+        db_user = ask("Enter the DB user for the #{env} environment:", default: "deployer")
+        puts
+
+        puts "Connection settings:".colorize(:green)
+        host = ask("Enter the IP or hostname of the DB server:", default: "myawesomeapp.com")
+        dump_type = ask("Should the dump happen over a ssh connection or can pg_dump access the DB port directly? (if the DB lives on a seperat server pg_dump the way to go)", default: "pg_dump", options: ["ssh", "pg_dump"])
+
+        ssh_user, ssh_port, postgres_port = nil
+        if dump_type == "ssh"
+          ssh_user = ask("Enter the username for the SSH connection:", default: "deployer")
+          ssh_port = ask("Enter the port for the SSH connection:", default: "22")
+        else
+          postgres_port = ask("Enter the database port for the pg_dump command:", default: "5432")
+        end
+        puts
+
+        env_config = {
+          env.to_s => {
+            "database" => {
+              "name" => db_name,
+              "user" => db_user
+            },
+            "connection" => {
+              "host" => host,
+              "dump_type" => dump_type,
+              "postgres_port" => postgres_port,
+              "ssh_user" => ssh_user,
+              "ssh_port" => ssh_port
+            }
+          }
+        }
+        @config.append(env_config, to: :environments)
+
+        continue = ask("Do you wanna add another environment? (anything other than 'yes' will exit)")
+        if continue&.downcase == "yes"
+          environment_count += 1
+        else
+          enter_new_environments = false
+        end
+      end
+
+      @config.write
+    end
+
+    # TODO: validate user input
+    # private
+    # def validate_config(config)
+    #   config.each do |key, value|
+    #
+    #   end
+    # end
+  end
+end

data/lib/remote_database_importer/operation.rb

@@ -0,0 +1,112 @@
+module RemoteDatabaseImporter
+  class Operation
+    require "remote_database_importer/config"
+    require "tty/spinner/multi"
+    require "pry"
+
+    LOG_FILE = "tmp/remote_database_importer.log"
+
+    def initialize
+      config_handler = RemoteDatabaseImporter::Config.new
+      @config = config_handler.read_or_create_configfile
+    end
+
+    def environments
+      @config.fetch("environments")
+    end
+
+    def select_environment
+      if environments.size > 1
+        puts "Select the operation environment:"
+
+        environments.map(&:keys).flatten.each_with_index do |env, index|
+          puts "#{index} for #{env.capitalize}"
+        end
+        env = environments[$stdin.gets.chomp.to_i].values[0]
+        raise "Environment couldn't be found!" if env.blank?
+        @current_environment = env
+        return
+      end
+
+      @current_environment = environments[0].values[0]
+    end
+
+    def import
+      select_environment
+      multi_spinner = TTY::Spinner::Multi.new("[:spinner] Import remote DB", format: :dots_3)
+      tasks = create_tasks_and_spinners(multi_spinner)
+
+      puts "Be aware, you may get asked for a password for the ssh or db connection"
+      tasks.each do |task|
+        task[:spinner].auto_spin
+        task_execution_was_successful = system(task[:command])
+        return "Can't continue, following task failed: #{task[:command]} - checkout the logfile: #{LOG_FILE}" unless task_execution_was_successful
+        task[:spinner].stop("... Done!")
+      end
+    end
+
+    private
+
+    def create_tasks_and_spinners(multi_spinner)
+      tasks = [
+        {name: "Terminate current DB sessions", command: terminate_current_db_sessions},
+        {name: "Dump remote DB", command: dump_remote_db},
+        {name: "Drop and create local DB", command: drop_and_create_local_db},
+        {name: "Restore remote DB", command: restore_db},
+        {name: "Run migrations", command: run_migrations},
+        {name: "Remove logfile", command: remove_logfile},
+        {name: "Remove dumpfile", command: remove_dumpfile}
+      ]
+      tasks.each.with_index(1) do |task, index|
+        task[:spinner] = multi_spinner.register "#{index}/#{tasks.length} :spinner #{task[:name]}"
+      end
+      tasks
+    end
+
+    # terminate local db sessions, otherwise the db can't be dropped
+    def terminate_current_db_sessions
+      "psql -d #{@config.fetch("local_db_name")} -c 'SELECT pg_terminate_backend(pg_stat_activity.pid) FROM pg_stat_activity WHERE datname = current_database() AND pid <> pg_backend_pid();' > #{LOG_FILE}"
+    end
+
+    def dump_remote_db
+      env = @current_environment
+      host = env["connection"]["host"]
+      db_name = env["database"]["name"]
+      db_user = env["database"]["user"]
+      dump_type = env["connection"]["dump_type"]
+      ssh_user = env["connection"]["ssh_user"]
+      ssh_port = env["connection"]["ssh_port"]
+      postgres_port = env["connection"]["postgres_port"]
+
+      if dump_type == "ssh"
+        "ssh #{ssh_user}@#{host} -p #{ssh_port} 'pg_dump -Fc -U #{db_user} -d #{db_name} -h localhost -C' > #{db_dump_location}"
+      else
+        "pg_dump -Fc 'host=#{host} dbname=#{db_name} user=#{db_user} port=#{postgres_port}' > #{db_dump_location}"
+      end
+    end
+
+    def drop_and_create_local_db
+      "rails db:environment:set RAILS_ENV=development; rake db:drop db:create > #{LOG_FILE}"
+    end
+
+    def restore_db
+      "pg_restore --jobs 8 --no-privileges --no-owner --dbname #{@config.fetch("local_db_name")} #{db_dump_location}"
+    end
+
+    def run_migrations
+      "rake db:migrate > #{LOG_FILE}"
+    end
+
+    def remove_logfile
+      "rm #{LOG_FILE}"
+    end
+
+    def remove_dumpfile
+      "rm #{db_dump_location}"
+    end
+
+    def db_dump_location
+      "tmp/#{@current_environment["database"]["name"]}.dump"
+    end
+  end
+end

data/lib/remote_database_importer/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module RemoteDatabaseImporter
+  VERSION = "0.1.0"
+end

data/lib/remote_database_importer.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require_relative "remote_database_importer/version"
+require_relative "remote_database_importer/operation"
+
+module RemoteDatabaseImporter
+  class Error < StandardError; end
+  require_relative "railtie" if defined?(Rails)
+end

data/lib/tasks/remote_database_importer.rake

@@ -0,0 +1,7 @@
+namespace :remote_database do
+  desc "Pulls a database to your filesystem"
+  task import: :environment do
+    importer = RemoteDatabaseImporter::Operation.new
+    importer.import
+  end
+end

data/sig/remote_database_importer.rbs

@@ -0,0 +1,4 @@
+module RemoteDatabaseImporter
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end

metadata

@@ -0,0 +1,137 @@
+--- !ruby/object:Gem::Specification
+name: remote_database_importer
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Leon
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2022-10-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: tty-config
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: tty-spinner
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.14'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.14'
+description: Dump remote database and import locally. Currently only postgres databases
+  supported
+email:
+- nonick@nonick.ch
+executables:
+- remote_database_importer
+extensions: []
+extra_rdoc_files: []
+files:
+- ".rspec"
+- ".standard.yml"
+- CHANGELOG.md
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/remote_database_importer
+- lib/railtie.rb
+- lib/remote_database_importer.rb
+- lib/remote_database_importer/config.rb
+- lib/remote_database_importer/operation.rb
+- lib/remote_database_importer/version.rb
+- lib/tasks/remote_database_importer.rake
+- readme_images/.DS_Store
+- readme_images/config_sample.png
+- readme_images/import-job.gif
+- sig/remote_database_importer.rbs
+homepage: https://github.com/leon-vogt/remote_database_importer
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/leon-vogt/remote_database_importer
+  source_code_uri: https://github.com/leon-vogt/remote_database_importer
+  changelog_uri: https://github.com/leon-vogt/remote_database_importer/blob/main/CHANGELOG.md
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.6.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.7
+signing_key:
+specification_version: 4
+summary: Dump remote database and import locally
+test_files: []