cloner 0.13.0 → 0.14.0

data/lib/cloner/mysql.rb

@@ -1,17 +1,19 @@
 module Cloner::MySQL
+  extend ActiveSupport::Concern
+  
   def my_local_auth
-    if ar_conf['password'].blank?
+    if local_db_config['password'].blank?
       ""
     else
-      "--password='#{ar_conf['password']}'"
+      "--password='#{local_db_config['password']}'"
     end
   end
 
   def my_remote_auth
-    if ar_r_conf['password'].blank?
+    if remote_db_config['password'].blank?
       ""
     else
-      "--password='#{ar_r_conf['password']}'"
+      "--password='#{remote_db_config['password']}'"
     end
   end
 
@@ -26,6 +28,35 @@ module Cloner::MySQL
   def my_bin_path(util)
     util
   end
+  
+  def my_local_bin_path(util)
+    if local_docker_compose? && local_docker_compose_service
+      # For mysql restore, we pipe data in, so we don't wrap it
+      return util if util == "mysql"
+      
+      # Build docker compose exec command
+      compose_cmd = local_docker_compose_exec(
+        local_docker_compose_service, 
+        util,
+        no_tty: true
+      )
+      return compose_cmd
+    end
+    my_bin_path(util)
+  end
+  
+  def my_remote_bin_path(util)
+    if remote_docker_compose? && remote_docker_compose_service
+      # Build docker compose exec command for remote
+      compose_cmd = remote_docker_compose_exec(
+        remote_docker_compose_service,
+        util,
+        no_tty: true
+      )
+      return compose_cmd
+    end
+    my_bin_path(util)
+  end
 
   def my_dump_remote
     puts "backup remote DB via ssh"
@@ -33,9 +64,9 @@ module Cloner::MySQL
       ssh.exec!("rm -R #{e remote_dump_path}")
       ret = ssh_exec!(ssh, "mkdir -p #{e remote_dump_path}")
       check_ssh_err(ret)
-      host = ar_r_conf['host'].present? ? " --host #{e ar_r_conf['host']}" : ""
-      port = ar_r_conf['port'].present? ? " --port #{e ar_r_conf['port']}" : ""
-      dump = "#{my_bin_path 'mysqldump'} #{my_dump_param} --user #{e ar_r_conf['username']} #{my_remote_auth}#{host}#{port} #{e ar_r_conf['database']} > #{e(remote_dump_path + '/'+db_file_name+'.sql')}"
+      host = remote_db_config['host'].present? ? " --host #{e remote_db_config['host']}" : ""
+      port = remote_db_config['port'].present? ? " --port #{e remote_db_config['port']}" : ""
+      dump = "#{my_remote_bin_path 'mysqldump'} #{my_dump_param} --user #{e remote_db_config['username']} #{my_remote_auth}#{host}#{port} #{e remote_db_config['database']} > #{e(remote_dump_path + '/'+db_file_name+'.sql')}"
       puts dump if verbose?
       ret = ssh_exec!(ssh, dump)
       check_ssh_err(ret)
@@ -44,15 +75,34 @@ module Cloner::MySQL
 
   def my_dump_restore
     puts "restoring DB"
-    host = ar_conf['host'].present? ? " --host #{e ar_conf['host']}" : ""
-    port = ar_conf['port'].present? ? " --port #{e ar_conf['port']}" : ""
-    restore = "#{my_bin_path 'mysql'} #{my_restore_param} --user #{e ar_conf['username']} #{my_local_auth}#{host}#{port} #{e ar_to} < #{e(my_path + '/'+db_file_name+'.sql')}"
-    puts restore if verbose?
-    pipe = IO.popen(restore)
-    while (line = pipe.gets)
-      print line if verbose?
+    
+    if local_docker_compose? && local_docker_compose_service
+      # Docker compose restore - pipe the SQL file to docker compose exec
+      host = local_db_config['host'].present? ? " --host #{e local_db_config['host']}" : ""
+      port = local_db_config['port'].present? ? " --port #{e local_db_config['port']}" : ""
+      
+      compose_path = local_docker_compose_path
+      compose_file = local_docker_compose_file
+      service = local_docker_compose_service
+      
+      # MySQL requires password to be passed differently in Docker
+      restore = "cat #{e(my_path + '/'+db_file_name+'.sql')} | (cd #{e compose_path} && docker compose -f #{e compose_file} exec -T #{e service} mysql #{my_restore_param} --user #{e local_db_config['username']} #{my_local_auth}#{host}#{port} #{e ar_to})"
+      puts restore if verbose?
+      system(restore)
+      ret = $?.to_i
+    else
+      # Standard restore
+      host = local_db_config['host'].present? ? " --host #{e local_db_config['host']}" : ""
+      port = local_db_config['port'].present? ? " --port #{e local_db_config['port']}" : ""
+      restore = "#{my_local_bin_path 'mysql'} #{my_restore_param} --user #{e local_db_config['username']} #{my_local_auth}#{host}#{port} #{e ar_to} < #{e(my_path + '/'+db_file_name+'.sql')}"
+      puts restore if verbose?
+      pipe = IO.popen(restore)
+      while (line = pipe.gets)
+        print line if verbose?
+      end
+      ret = $?.to_i
     end
-    ret = $?.to_i
+    
     if ret != 0
       puts "Error: local command exited with #{ret}"
     end

data/lib/cloner/postgres.rb

@@ -2,18 +2,18 @@ module Cloner::Postgres
   extend ActiveSupport::Concern
 
   def pg_local_auth
-    if ar_conf['password'].blank?
+    if local_db_config['password'].blank?
       ""
     else
-      "PGPASSWORD='#{ar_conf['password']}' "
+      "PGPASSWORD='#{local_db_config['password']}' "
     end
   end
 
   def pg_remote_auth
-    if ar_r_conf['password'].blank?
+    if remote_db_config['password'].blank?
       ""
     else
-      "PGPASSWORD='#{ar_r_conf['password']}' "
+      "PGPASSWORD='#{remote_db_config['password']}' "
     end
   end
 
@@ -37,10 +37,39 @@ module Cloner::Postgres
   end
 
   def pg_local_bin_path(util)
+    if local_docker_compose? && local_docker_compose_service
+      # For pg_restore, we pipe data in, so we don't wrap it in docker compose
+      return util if util == "pg_restore"
+      
+      # Build docker compose exec command
+      env_vars = {}
+      env_vars['PGPASSWORD'] = local_db_config['password'] if local_db_config['password'].present?
+      
+      compose_cmd = local_docker_compose_exec(
+        local_docker_compose_service, 
+        "env PGPASSWORD='#{local_db_config['password']}' #{util}",
+        env: env_vars,
+        no_tty: true
+      )
+      return compose_cmd
+    end
     pg_bin_path(util)
   end
   
   def pg_remote_bin_path(util)
+    if remote_docker_compose? && remote_docker_compose_service
+      # Build docker compose exec command for remote
+      env_vars = {}
+      env_vars['PGPASSWORD'] = remote_db_config['password'] if remote_db_config['password'].present?
+      
+      compose_cmd = remote_docker_compose_exec(
+        remote_docker_compose_service,
+        "env PGPASSWORD='#{remote_db_config['password']}' #{util}",
+        env: env_vars,
+        no_tty: true
+      )
+      return compose_cmd
+    end
     pg_bin_path(util)
   end 
 
@@ -50,9 +79,9 @@ module Cloner::Postgres
       ssh.exec!("rm -R #{e remote_dump_path}")
       ret = ssh_exec!(ssh, "mkdir -p #{e remote_dump_path}")
       check_ssh_err(ret)
-      host = ar_r_conf['host'].present? ? " -h #{e ar_r_conf['host']}" : ""
-      port = ar_r_conf['port'].present? ? " -p #{e ar_r_conf['port']}" : ""
-      dump = pg_remote_auth + "#{pg_remote_bin_path 'pg_dump'} #{pg_dump_param} -U #{e ar_r_conf['username']}#{host}#{port} #{e ar_r_conf['database']} > #{e(remote_dump_path + '/'+db_file_name+'.bak')}"
+      host = remote_db_config['host'].present? ? " -h #{e remote_db_config['host']}" : ""
+      port = remote_db_config['port'].present? ? " -p #{e remote_db_config['port']}" : ""
+      dump = pg_remote_auth + "#{pg_remote_bin_path 'pg_dump'} #{pg_dump_param} -U #{e remote_db_config['username']}#{host}#{port} #{e remote_db_config['database']} > #{e(remote_dump_path + '/'+db_file_name+'.bak')}"
       puts dump if verbose?
       ret = ssh_exec!(ssh, dump)
       check_ssh_err(ret)
@@ -61,15 +90,34 @@ module Cloner::Postgres
 
   def pg_dump_restore
     puts "restoring DB"
-    host = ar_conf['host'].present? ? " -h #{e ar_conf['host']}" : ""
-    port = ar_conf['port'].present? ? " -p #{e ar_conf['port']}" : ""
-    restore = pg_local_auth + "#{pg_local_bin_path 'pg_restore'} #{pg_restore_param} -U #{e ar_conf['username']}#{host}#{port} -d #{e ar_to} #{e(pg_path + '/'+db_file_name+'.bak')}"
-    puts restore if verbose?
-    pipe = IO.popen(restore)
-    while (line = pipe.gets)
-      print line if verbose?
+    
+    if local_docker_compose? && local_docker_compose_service
+      # Docker compose restore - pipe the backup file to docker compose exec
+      host = local_db_config['host'].present? ? " -h #{e local_db_config['host']}" : ""
+      port = local_db_config['port'].present? ? " -p #{e local_db_config['port']}" : ""
+      
+      env_str = local_db_config['password'].present? ? "--env PGPASSWORD=#{e local_db_config['password']}" : ""
+      compose_path = local_docker_compose_path
+      compose_file = local_docker_compose_file
+      service = local_docker_compose_service
+      
+      restore = "cat #{e(pg_path + '/'+db_file_name+'.bak')} | (cd #{e compose_path} && docker compose -f #{e compose_file} exec -T #{env_str} #{e service} pg_restore #{pg_restore_param} -U #{e local_db_config['username']}#{host}#{port} -d #{e ar_to})"
+      puts restore if verbose?
+      system(restore)
+      ret = $?.to_i
+    else
+      # Standard restore
+      host = local_db_config['host'].present? ? " -h #{e local_db_config['host']}" : ""
+      port = local_db_config['port'].present? ? " -p #{e local_db_config['port']}" : ""
+      restore = pg_local_auth + "#{pg_local_bin_path 'pg_restore'} #{pg_restore_param} -U #{e local_db_config['username']}#{host}#{port} -d #{e ar_to} #{e(pg_path + '/'+db_file_name+'.bak')}"
+      puts restore if verbose?
+      pipe = IO.popen(restore)
+      while (line = pipe.gets)
+        print line if verbose?
+      end
+      ret = $?.to_i
     end
-    ret = $?.to_i
+    
     if ret != 0
       puts "Error: local command exited with #{ret}"
     end

data/lib/cloner/version.rb

@@ -1,4 +1,4 @@
 module Cloner
-  VERSION = "0.13.0"
+  VERSION = "0.14.0"
 end
 

data/lib/cloner.rb

@@ -6,6 +6,7 @@ require 'fileutils'
 module Cloner
   autoload :Base, "cloner/base"
   autoload :Internal, "cloner/internal"
+  autoload :DockerCompose, "cloner/docker_compose"
   autoload :Ar, "cloner/ar"
   autoload :MongoDB, "cloner/mongodb"
   autoload :Postgres, "cloner/postgres"

data/lib/generators/cloner_generator.rb

@@ -2,13 +2,16 @@ class ClonerGenerator < Rails::Generators::Base
   source_root File.expand_path('templates', __dir__)
 
   class_option :extend, default: false, type: :boolean, aliases: '-e'
+  class_option :docker_compose, default: false, type: :boolean, aliases: '-d', desc: 'Include Docker Compose configuration examples'
 
   desc "This generator create lib/tasks/dl.thor"
   def create_task_file
-    unless options[:extend]
-      create_default_task_file
-    else
+    if options[:docker_compose]
+      create_docker_compose_task_file
+    elsif options[:extend]
       create_extended_task_file
+    else
+      create_default_task_file
     end
   end
 
@@ -19,7 +22,18 @@ class ClonerGenerator < Rails::Generators::Base
 
   def create_extended_task_file
     say 'Create extend file'
-    @username = Rails.application.class.parent_name.downcase
+    module_parent_name = if Rails::VERSION::MAJOR >= 6
+                           Rails.application.class.module_parent_name
+                         else
+                           Rails.application.class.parent_name
+                         end
+    @username = module_parent_name.downcase
     template 'cloner_extend.thor.erb', 'lib/tasks/dl.thor'
   end
+  
+  def create_docker_compose_task_file
+    say 'Create Docker Compose file with examples'
+    @username = Rails.application.class.parent_name.downcase
+    template 'cloner_docker_compose.thor.erb', 'lib/tasks/dl.thor'
+  end
 end

data/lib/generators/templates/cloner_docker_compose.thor.erb

@@ -0,0 +1,154 @@
+require 'cloner'
+
+class Dl < Cloner::Base
+  no_commands do
+    def rails_path
+      File.expand_path("../../../config/environment", __FILE__)
+    end
+    
+    def stages
+      @_stages ||= {
+        # TODO: Add new stages here if you needed
+        production: {
+          # TODO: Fix production settings
+          ssh_host: 'production.example.com',
+          ssh_user: '<%= @username %>',
+          # Docker Compose settings for remote
+          docker_compose: true,
+          docker_compose_service: 'db',  # service name in compose.yml
+          docker_compose_path: '/data/<%= @username %>/app/current'
+        },
+        staging: {
+          # TODO: Fix staging settings
+          ssh_host: 'staging.example.com',
+          ssh_user: '<%= @username %>',
+          # Docker Compose settings for remote
+          docker_compose: true,
+          docker_compose_service: 'db',
+          docker_compose_path: '/data/<%= @username %>/app/current'
+        }
+      }
+    end
+    
+    def ssh_host
+      stages.dig(options[:from].to_sym, :ssh_host)
+    end
+    
+    def ssh_user
+      stages.dig(options[:from].to_sym, :ssh_user)
+    end
+    
+    def remote_dump_path
+      # TODO: Fix remote dump path
+      '/data/<%= @username %>/dump'
+    end
+    
+    def remote_app_path
+      # TODO: Fix remote app path
+      '/data/<%= @username %>/app/current'
+    end
+    
+    # Docker Compose configuration for remote
+    def remote_docker_compose?
+      stages.dig(options[:from].to_sym, :docker_compose) || false
+    end
+    
+    def remote_docker_compose_service
+      stages.dig(options[:from].to_sym, :docker_compose_service)
+    end
+    
+    def remote_docker_compose_path
+      stages.dig(options[:from].to_sym, :docker_compose_path) || remote_app_path
+    end
+    
+    # Docker Compose configuration for local (if needed)
+    def local_docker_compose?
+      # TODO: Set to true if you use Docker Compose locally
+      false
+    end
+    
+    def local_docker_compose_service
+      # TODO: Set your local Docker Compose service name
+      'db'
+    end
+    
+    def local_docker_compose_path
+      # TODO: Set your local Docker Compose path
+      Rails.root.to_s
+    end
+    
+    # Override for PostgreSQL with Docker Compose (example)
+    # def pg_remote_bin_path(util)
+    #   if remote_docker_compose? && remote_docker_compose_service
+    #     # Example: Read .env file for credentials
+    #     env_vars = read_remote_env_file
+    #     "cd #{e remote_docker_compose_path} && docker compose exec --no-TTY --env PGPASSWORD='#{env_vars['DB_PASSWORD']}' #{e remote_docker_compose_service} #{util}"
+    #   else
+    #     super
+    #   end
+    # end
+    
+    # Helper to read remote .env file (example)
+    # def read_remote_env_file
+    #   env_content = ""
+    #   do_ssh do |ssh|
+    #     env_content = ssh.exec!("cat #{e remote_app_path}/.env")
+    #   end
+    #   
+    #   # Parse .env content
+    #   env_vars = {}
+    #   env_content.each_line do |line|
+    #     next if line.strip.empty? || line.strip.start_with?('#')
+    #     key, value = line.strip.split('=', 2)
+    #     next unless key && value
+    #     # Remove quotes if present
+    #     value = value.gsub(/^["']|["']$/, '')
+    #     env_vars[key] = value
+    #   end
+    #   env_vars
+    # end
+  end
+
+  class_option :from,
+               default: 'production',
+               type: :string,
+               desc: 'stage name where cloner get data'
+  class_option :skip_database,
+               default: false,
+               type: :boolean,
+               aliases: '-D',
+               desc: 'skip clone database'
+  class_option :skip_files,
+               default: false,
+               type: :boolean,
+               aliases: '-F',
+               desc: 'skip clone files'
+
+  desc "download", "clone files and DB from production"
+  def download
+    load_env
+    say "Clone from: #{options[:from]}", :green
+    
+    if remote_docker_compose?
+      say "Using Docker Compose on remote (service: #{remote_docker_compose_service})", :blue
+    end
+    
+    if local_docker_compose?
+      say "Using Docker Compose locally (service: #{local_docker_compose_service})", :blue
+    end
+    
+    if options[:skip_database]
+      say "Skip clone database!", :yellow
+    else
+      clone_db
+    end
+
+    if options[:skip_files]
+      say "Skip clone files!", :yellow
+    else
+      # TODO: Fix folders for synchronization here
+      rsync_public("ckeditor_assets")
+      rsync_public("uploads")
+    end
+  end
+end

data/test/dl_compose_test.thor

@@ -0,0 +1,158 @@
+require_relative '../lib/cloner'
+require 'pathname'
+require 'net/ssh'
+require 'shellwords'
+require 'active_support/core_ext/object'
+
+class ::Rails
+  def self.root
+    Pathname.new(File.expand_path("../", __FILE__))
+  end
+  def self.env
+    "development"
+  end
+end
+
+class DlComposeTest < Cloner::Base
+  no_commands do
+    def rails_path
+      File.expand_path("../", __FILE__)
+    end
+
+    def load_env
+      require 'net/ssh'
+      require 'shellwords'
+      require 'active_support/core_ext/object'
+    end
+
+    # SSH configuration
+    def ssh_host
+      'rscx.ru'
+    end
+
+    def ssh_user
+      'root'
+    end
+
+    def env_from
+      "production"
+    end
+
+    # Remote paths
+    def remote_dump_path
+      '/root/compose/aichaos/tmp_dump'
+    end
+
+    def remote_app_path
+      '/root/compose/aichaos'
+    end
+
+    # Docker Compose configuration for remote
+    def remote_docker_compose?
+      true
+    end
+
+    def remote_docker_compose_service
+      'db'  # Assuming the PostgreSQL service is named 'db'
+    end
+
+    def remote_docker_compose_path
+      remote_app_path
+    end
+
+    # Docker Compose configuration for local
+    def local_docker_compose?
+      true
+    end
+
+    def local_docker_compose_service
+      'testdb'
+    end
+
+    def local_docker_compose_path
+      Rails.root.parent.to_s
+    end
+
+    # Override to read credentials from remote .env file
+    def read_ar_r_conf
+      # Read from remote .env file
+      env_content = ""
+      do_ssh do |ssh|
+        env_content = ssh.exec!("cat #{e remote_app_path}/.env")
+      end
+      
+      # Parse .env content
+      env_vars = {}
+      env_content.each_line do |line|
+        next if line.strip.empty? || line.strip.start_with?('#')
+        key, value = line.strip.split('=', 2)
+        next unless key && value
+        # Remove quotes if present
+        value = value.gsub(/^["']|["']$/, '')
+        env_vars[key] = value
+      end
+      
+      {
+        adapter: "postgresql",
+        host: env_vars['DB_HOST'] || 'db',
+        port: env_vars['DB_PORT'] || '5432',
+        database: env_vars['DB_NAME'] || 'aichaos',
+        username: env_vars['DB_USER'] || 'aichaos',
+        password: env_vars['DB_PASSWORD'] || ''
+      }.stringify_keys
+    end
+
+    # Local database configuration
+    def ar_conf
+      {
+        adapter: "postgresql",
+        host: "localhost",
+        port: "5432",
+        database: "test_development",
+        username: "testuser",
+        password: "testpass"
+      }.stringify_keys
+    end
+
+    def ar_to
+      ar_conf['database']
+    end
+
+    def db_file_name
+      "cloner_test"
+    end
+
+    # Override pg_bin_path to handle Docker Compose
+    def pg_remote_bin_path(util)
+      if remote_docker_compose? && remote_docker_compose_service
+        env_vars = read_ar_r_conf
+        "cd #{e remote_app_path} && docker compose exec --no-TTY --env PGPASSWORD='#{env_vars['password']}' #{e remote_docker_compose_service} #{util}"
+      else
+        super
+      end
+    end
+
+    def verbose?
+      true
+    end
+  end
+
+  desc "test", "Test cloning from remote Docker Compose PostgreSQL"
+  def test
+    load_env
+    
+    puts "Testing Docker Compose PostgreSQL cloning..."
+    puts "Remote: #{ssh_user}@#{ssh_host}:#{remote_app_path}"
+    puts "Local: #{local_docker_compose_path}"
+    
+    # Start local Docker Compose
+    puts "\nStarting local Docker Compose..."
+    system("cd #{e local_docker_compose_path} && docker compose up -d")
+    sleep 5  # Wait for database to be ready
+    
+    # Clone the database
+    clone_db
+    
+    puts "\nTest completed!"
+  end
+end