cloner 0.13.0 → 0.14.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 688e6f12b2564228e07d5dec84b14a1cdaec1604c1d42a068217489f799a1277
-  data.tar.gz: b6ee8c6a77806f52d93db7489f69d9743370ff0de38e81c7a11e5a9b939a1965
+  metadata.gz: e2bcb2230fc707f0e164ad4a7e6462d35cc96889905940ebe882eba396530516
+  data.tar.gz: 8824290ff01994056936a731d1a5e35fa188abc20188b47f9b7dee23b6173b63
 SHA512:
-  metadata.gz: da35a85ce9e5229ca1ebbb0f134b686f8b2b3802489f8d3e97f30e3cc9ff60f84df233b72a35c832437fb82b36c0e4c2513e52ab436add8198296096bdf3ce80
-  data.tar.gz: 2bb381ab3429508fd9097502956345e146fc13c5b5fa6bda32a7189237c876d4af256203ac133910c80feec8115f0ece73dd5ac05b1092e5e5de10d3dbf3dfce
+  metadata.gz: 3a42dd7db0a91b1fa6bf2d2e6421914b823f982ddd89cdd36c76130bbebbb4f555afc667abc345fead697e26465599bf6ba5ab7dd104ffc4121f3911c4c9a6b1
+  data.tar.gz: eb889b20bd8a91b6cab12d367c17615ba4123b429cdf9703a7315ca7b7a8ec195e7fa69fcddb024b4a552bda9e9c58ff5e98ec0a9f5c8a798e4a6d8e09f8928a

data/.env

@@ -0,0 +1,5 @@
+DB_HOST=localhost
+DB_PORT=5432
+DB_NAME=test_db
+DB_USER=test_user
+DB_PASSWORD=test_pass

data/.ruby-version

@@ -1 +1 @@
-2.7.4
+3.4.5

data/CLAUDE.md

@@ -0,0 +1,58 @@
+# CLAUDE.md
+
+This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository.
+
+## About Cloner
+
+Cloner is a Ruby gem that simplifies cloning production databases and files to local development or staging environments. It supports MongoDB (via Mongoid), PostgreSQL, and MySQL databases, and uses rsync for file synchronization.
+
+## Key Commands
+
+```bash
+# Install dependencies
+bundle install
+
+# Generate cloner template (basic)
+bundle exec rails generate cloner
+
+# Generate extended template with more options
+bundle exec rails generate cloner -e
+
+# Run cloning (after generating template)
+thor dl
+
+# Clone only database (skip files)
+thor dl -F
+
+# Clone only files (skip database)
+thor dl -D
+```
+
+## Architecture Overview
+
+The gem is organized as a modular Thor-based CLI tool:
+
+1. **Core Module Structure** (`lib/cloner/`):
+   - `base.rb`: Thor command base class that users extend
+   - `internal.rb`: Core functionality shared across all adapters
+   - Database adapters: `mongodb.rb`, `mysql.rb`, `postgres.rb`, `ar.rb`
+   - `rsync.rb`: File synchronization logic
+   - `ssh.rb`: SSH connection handling
+
+2. **Rails Integration**:
+   - Generator in `lib/generators/cloner_generator.rb` creates `lib/tasks/dl.thor`
+   - Templates in `lib/generators/templates/` provide base and extended configurations
+
+3. **Key Design Patterns**:
+   - Uses Thor for CLI interface
+   - Modular adapters for different database types
+   - Override methods for customization (e.g., `ssh_host`, `ssh_user`, `remote_dump_path`)
+   - Leverages native database tools (pg_dump, mysqldump, mongodump)
+
+## Development Notes
+
+- The gem version is defined in `lib/cloner/version.rb`
+- Uses `fileutils` for file operations (ensure compatibility)
+- Supports Rails 6+ multi-database configurations
+- SSH connections use net-ssh gem
+- Database detection automatically chooses appropriate adapter based on Rails configuration

data/README.md

@@ -107,8 +107,174 @@ All functions from cloner/internal.rb can be overriden, for example:
       {}
     end
 
+## Docker Compose Support
+
+Cloner now supports Docker Compose for both local and remote database operations. This is useful when your databases run inside Docker containers.
+
+### Generating Docker Compose Template
+
+To generate a template with Docker Compose examples:
+
+```
+bundle exec rails generate cloner -d
+```
+
+### Configuration
+
+Add these methods to your `dl.thor` file to enable Docker Compose:
+
+```ruby
+# For remote Docker Compose
+def remote_docker_compose?
+  true
+end
+
+def remote_docker_compose_service
+  'db'  # Your database service name in compose.yml
+end
+
+def remote_docker_compose_path
+  remote_app_path  # Path where compose.yml is located
+end
+
+# Optional: Override compose file name (default is 'compose.yml')
+def remote_docker_compose_file
+  'docker-compose.yml'  # Use if your file is named differently
+end
+
+# For local Docker Compose
+def local_docker_compose?
+  true
+end
+
+def local_docker_compose_service
+  'db'  # Your local database service name
+end
+
+def local_docker_compose_path
+  Rails.root.to_s  # Path where your local compose.yml is located
+end
+
+# Optional: Override compose file name for local (default is 'compose.yml')
+def local_docker_compose_file
+  'docker-compose.yml'  # Use if your file is named differently
+end
+
+# Optional: Override local database config (reads from local .env by default)
+def local_db_config
+  {
+    adapter: 'postgresql',
+    host: read_local_env('DB_HOST') || 'localhost',
+    port: read_local_env('DB_PORT') || '5432',
+    database: read_local_env('DB_NAME'),
+    username: read_local_env('DB_USER'),
+    password: read_local_env('DB_PASSWORD')
+  }.stringify_keys
+end
+```
+
+### PostgreSQL with Docker Compose Example
+
+```ruby
+class Dl < Cloner::Base
+  no_commands do
+    def ssh_host
+      'production.example.com'
+    end
+    
+    def ssh_user
+      'deploy'
+    end
+    
+    def remote_app_path
+      '/home/deploy/myapp'
+    end
+    
+    def remote_dump_path
+      "#{remote_app_path}/tmp_dump"
+    end
+    
+    # Enable Docker Compose for remote
+    def remote_docker_compose?
+      true
+    end
+    
+    def remote_docker_compose_service
+      'postgres'
+    end
+    
+    # Override to read credentials from .env file
+    def read_ar_r_conf
+      # Read from remote .env file
+      env_content = ""
+      do_ssh do |ssh|
+        env_content = ssh.exec!("cat #{e remote_app_path}/.env")
+      end
+      
+      # Parse .env content
+      env_vars = {}
+      env_content.each_line do |line|
+        next if line.strip.empty? || line.strip.start_with?('#')
+        key, value = line.strip.split('=', 2)
+        next unless key && value
+        value = value.gsub(/^["']|["']$/, '')
+        env_vars[key] = value
+      end
+      
+      {
+        adapter: "postgresql",
+        host: env_vars['DB_HOST'] || 'postgres',
+        database: env_vars['DB_NAME'],
+        username: env_vars['DB_USER'],
+        password: env_vars['DB_PASSWORD']
+      }.stringify_keys
+    end
+    
+    # Enable Docker Compose for local
+    def local_docker_compose?
+      true
+    end
+    
+    def local_docker_compose_service
+      'db'
+    end
+  end
+  
+  desc "download", "clone DB from production"
+  def download
+    load_env
+    clone_db
+  end
+end
+```
+
+### How It Works
+
+When Docker Compose is enabled:
+
+1. **Remote operations**: Database dump commands are wrapped with `docker compose exec` on the remote server
+2. **Local operations**: Database restore commands pipe data into `docker compose exec -T` locally
+3. **Automatic command wrapping**: The gem automatically detects and wraps database commands appropriately
+
+### Supported Databases
+
+Docker Compose support is available for:
+- PostgreSQL
+- MySQL
+
 ## Changelog
 
+### 0.14.0
+
+- Add Docker Compose support for local and remote database operations
+- Add Docker Compose generator template with `-d` option
+- Support automatic command wrapping for PostgreSQL and MySQL when using Docker Compose
+- Add helper methods for Docker Compose configuration
+- Default compose file name is now 'compose.yml' (configurable via docker_compose_file methods)
+- Add `local_db_config` and `remote_db_config` methods for customizing database configurations
+- Support reading from .env files for both local and remote environments
+- Replace direct usage of `ar_conf` and `ar_r_conf` with configurable `local_db_config` and `remote_db_config`
+
 ### 0.10.0
 
 - Support rails 6 multi database activerecord apps via option

data/compose.yml

@@ -0,0 +1,15 @@
+version: '3.8'
+
+services:
+  testdb:
+    restart: unless-stopped
+    image: postgres:17.4-alpine
+    container_name: testdb
+    environment:
+      - POSTGRES_USER=testuser
+      - POSTGRES_PASSWORD=testpass
+      - POSTGRES_DB=test_development
+    ports:
+      - "5432:5432"
+    volumes:
+      - ./db:/var/lib/postgresql/data

data/lib/cloner/ar.rb

@@ -1,7 +1,7 @@
 module Cloner::Ar
   def read_ar_conf
     @conf ||= begin
-      YAML.load(ERB.new(File.read(Rails.root.join('config', 'database.yml'))).result)[env_database]
+      YAML.load(ERB.new(File.read(Rails.root.join('config', 'database.yml'))).result, aliases: true)[env_database]
     end
   end
   def ar_conf
@@ -26,7 +26,7 @@ module Cloner::Ar
   end
 
   def ar_to
-    ar_conf['database']
+    local_db_config['database']
   end
 
   def read_ar_r_conf
@@ -35,7 +35,7 @@ module Cloner::Ar
         ret = ssh_exec!(ssh, "cat #{e(remote_app_path + '/config/database.yml')}")
         check_ssh_err(ret)
         begin
-          res = YAML.load(ERB.new(ret[0]).result)[env_from]
+          res = YAML.load(ERB.new(ret[0]).result, aliases: true)[env_from]
           raise 'no data' if res.blank?
           #res['host'] ||= '127.0.0.1'
         rescue Exception => e
@@ -65,19 +65,19 @@ module Cloner::Ar
   end
 
   def run_clone_ar
-    if ar_conf["adapter"] != ar_r_conf["adapter"]
-      puts "Error: ActiveRecord adapter mismatch: local #{ar_conf["adapter"]}, remote #{ar_r_conf["adapter"]}"
+    if local_db_config["adapter"] != remote_db_config["adapter"]
+      puts "Error: ActiveRecord adapter mismatch: local #{local_db_config["adapter"]}, remote #{remote_db_config["adapter"]}"
       puts "it is not possible to convert from one database to another via this tool."
       exit
     end
 
-    case ar_conf["adapter"]
+    case local_db_config["adapter"]
     when 'postgresql'
       clone_pg
     when 'mysql2'
       clone_my
     else
-      puts "unknown activerecord adapter: #{ar_conf["adapter"]}"
+      puts "unknown activerecord adapter: #{local_db_config["adapter"]}"
       puts "currently supported adapters: mysql2, postgresql"
       exit
     end

data/lib/cloner/docker_compose.rb

@@ -0,0 +1,283 @@
+require 'active_support/core_ext/hash'
+
+module Cloner::DockerCompose
+  extend ActiveSupport::Concern
+
+  # Docker Compose configuration methods
+  def use_docker_compose?
+    false
+  end
+
+  def local_docker_compose?
+    false
+  end
+
+  def remote_docker_compose?
+    false
+  end
+
+  def docker_compose_file
+    "compose.yml"
+  end
+
+  def local_docker_compose_file
+    docker_compose_file
+  end
+
+  def remote_docker_compose_file
+    docker_compose_file
+  end
+
+  def docker_compose_service
+    nil
+  end
+
+  def local_docker_compose_service
+    docker_compose_service
+  end
+
+  def remote_docker_compose_service
+    docker_compose_service
+  end
+
+  def docker_compose_path
+    "."
+  end
+
+  def local_docker_compose_path
+    docker_compose_path
+  end
+
+  def remote_docker_compose_path
+    remote_app_path
+  end
+
+  def docker_compose_exec(service, command, opts = {})
+    compose_file = opts[:compose_file] || docker_compose_file
+    compose_path = opts[:compose_path] || docker_compose_path
+    no_tty = opts[:no_tty] != false ? "--no-TTY" : ""
+    env_vars = opts[:env] || {}
+    
+    env_str = env_vars.map { |k, v| "--env #{e k}=#{e v}" }.join(" ")
+    
+    "cd #{e compose_path} && docker compose -f #{e compose_file} exec #{no_tty} #{env_str} #{e service} #{command}"
+  end
+
+  def local_docker_compose_exec(service, command, opts = {})
+    opts[:compose_file] ||= local_docker_compose_file
+    opts[:compose_path] ||= local_docker_compose_path
+    docker_compose_exec(service, command, opts)
+  end
+
+  def remote_docker_compose_exec(service, command, opts = {})
+    opts[:compose_file] ||= remote_docker_compose_file
+    opts[:compose_path] ||= remote_docker_compose_path
+    docker_compose_exec(service, command, opts)
+  end
+
+  # Helper to wrap commands with docker compose when needed
+  def wrap_command(command, local: true)
+    if local && local_docker_compose?
+      service = local_docker_compose_service
+      return local_docker_compose_exec(service, command) if service
+    elsif !local && remote_docker_compose?
+      service = remote_docker_compose_service
+      return remote_docker_compose_exec(service, command) if service
+    end
+    command
+  end
+  
+  # Helper to read and parse remote .env file
+  def remote_env_content
+    @remote_env_content ||= begin
+      content = ""
+      do_ssh do |ssh|
+        # Use docker compose path, not app path
+        env_path = "#{remote_docker_compose_path}/.env"
+        # First check if file exists
+        check_cmd = "test -f #{e env_path} && echo 'EXISTS' || echo 'NOT_EXISTS'"
+        check_ret = ssh_exec!(ssh, check_cmd)
+        
+        if check_ret && check_ret[0] && check_ret[0].strip == 'NOT_EXISTS'
+          puts "ERROR: .env file not found at #{env_path}"
+          puts "Try running: ssh #{ssh_user}@#{ssh_host} 'ls -la #{remote_docker_compose_path}/'"
+          exit 1
+        end
+        
+        # Then read the file
+        read_cmd = "cat #{e env_path} 2>&1"
+        ret = ssh_exec!(ssh, read_cmd)
+        
+        # ssh_exec! returns [stdout, stderr, exit_code, ...]
+        content = ret[0] if ret && ret[0]
+        
+        if content.to_s.strip == 'FILE NOT FOUND'
+          puts "ERROR: .env file not found at #{env_path}"
+          puts "Try running: ssh #{ssh_user}@#{ssh_host} 'ls -la #{remote_docker_compose_path}/'"
+          exit 1
+        end
+      end
+      content || ""
+    end
+  end
+  
+  # Parse .env content into a hash
+  def remote_env_vars
+    @remote_env_vars ||= begin
+      vars = {}
+      content = remote_env_content
+      content.each_line.with_index do |line, idx|
+        original_line = line
+        line = line.strip
+        next if line.empty? || line.start_with?('#')
+        
+        # Handle KEY=VALUE format
+        if match = line.match(/^([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/)
+          key = match[1]
+          value = match[2]
+          
+          # Remove surrounding quotes if present
+          value = value.gsub(/^["']|["']$/, '') if value
+          
+          vars[key] = value
+        else
+        end
+      end
+      vars
+    end
+  end
+  
+  # Helper to read a specific env var from remote .env file
+  def read_remote_env(key = nil)
+    if key.nil?
+      # Return all env vars
+      remote_env_vars
+    else
+      # Return specific key
+      value = remote_env_vars[key]
+      value
+    end
+  end
+  
+  # Helper to read and parse local .env file
+  def local_env_content
+    @local_env_content ||= begin
+      # Use docker compose path for .env file
+      env_path = File.join(local_docker_compose_path, '.env')
+      if File.exist?(env_path)
+        content = File.read(env_path)
+        content
+      else
+        ""
+      end
+    end
+  end
+  
+  # Parse local .env content into a hash
+  def local_env_vars
+    @local_env_vars ||= begin
+      vars = {}
+      local_env_content.each_line do |line|
+        line = line.strip
+        next if line.empty? || line.start_with?('#')
+        
+        # Handle KEY=VALUE format
+        if match = line.match(/^([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$/)
+          key = match[1]
+          value = match[2]
+          
+          # Remove surrounding quotes if present
+          value = value.gsub(/^["']|["']$/, '') if value
+          
+          vars[key] = value
+        end
+      end
+      vars
+    end
+  end
+  
+  # Helper to read a specific env var from local .env file
+  def read_local_env(key = nil)
+    if key.nil?
+      # Return all env vars
+      local_env_vars
+    else
+      # Return specific key
+      value = local_env_vars[key]
+      value
+    end
+  end
+  
+  # Default local database config for Docker Compose
+  # Override this method to customize your database configuration
+  def local_db_config
+    if local_docker_compose?
+      # When using Docker Compose, read from .env file
+      config = {
+        adapter: 'postgresql',
+        host: read_local_env('DB_HOST') || 'localhost',
+        port: read_local_env('DB_PORT') || '5432',
+        database: read_local_env('DB_NAME'),
+        username: read_local_env('DB_USER'),
+        password: read_local_env('DB_PASSWORD') || ''
+      }.stringify_keys
+      
+      # Validate required fields
+      if config['database'].nil? || config['database'].empty?
+        puts "Error: DB_NAME not found in local .env file at #{local_docker_compose_path}/.env"
+        puts "Available env vars: #{local_env_vars.keys.join(', ')}"
+        puts "Local .env content (first 10 lines):"
+        puts local_env_content.lines.first(10).join
+        exit 1
+      end
+      
+      if config['username'].nil? || config['username'].empty?
+        puts "Error: DB_USER not found in local .env file at #{local_docker_compose_path}/.env"
+        puts "Available env vars: #{local_env_vars.keys.join(', ')}"
+        exit 1
+      end
+      
+      config
+    else
+      # Fall back to reading from database.yml
+      ar_conf
+    end
+  end
+  
+  # Default remote database config for Docker Compose
+  # Override this method to customize your database configuration
+  def remote_db_config
+    if remote_docker_compose?
+      # When using Docker Compose, read from .env file
+      config = {
+        adapter: 'postgresql',
+        host: read_remote_env('DB_HOST') || 'db',
+        port: read_remote_env('DB_PORT') || '5432',
+        database: read_remote_env('DB_NAME'),
+        username: read_remote_env('DB_USER'),
+        password: read_remote_env('DB_PASSWORD') || ''
+      }.stringify_keys
+      
+      
+      # Validate required fields
+      if config['database'].nil? || config['database'].empty?
+        puts "Error: DB_NAME not found in remote .env file at #{remote_docker_compose_path}/.env"
+        puts "Available env vars: #{remote_env_vars.keys.join(', ')}"
+        puts "Remote .env content (first 10 lines):"
+        puts remote_env_content.lines.first(10).join
+        exit 1
+      end
+      
+      if config['username'].nil? || config['username'].empty?
+        puts "Error: DB_USER not found in remote .env file at #{remote_docker_compose_path}/.env"
+        puts "Available env vars: #{remote_env_vars.keys.join(', ')}"
+        exit 1
+      end
+      
+      config
+    else
+      # Fall back to reading from database.yml
+      ar_r_conf
+    end
+  end
+end

data/lib/cloner/internal.rb

@@ -1,5 +1,6 @@
 module Cloner::Internal
   extend ActiveSupport::Concern
+  include Cloner::DockerCompose
   include Cloner::MongoDB
   include Cloner::Ar
   include Cloner::Postgres

data/lib/cloner/mongodb.rb

@@ -53,6 +53,36 @@ module Cloner::MongoDB
   def mongodb_dump_extra
     ""
   end
+  
+  def mongodb_bin_path(util)
+    util
+  end
+  
+  def mongodb_local_bin_path(util)
+    if local_docker_compose? && local_docker_compose_service
+      # Build docker compose exec command
+      compose_cmd = local_docker_compose_exec(
+        local_docker_compose_service, 
+        util,
+        no_tty: true
+      )
+      return compose_cmd
+    end
+    mongodb_bin_path(util)
+  end
+  
+  def mongodb_remote_bin_path(util)
+    if remote_docker_compose? && remote_docker_compose_service
+      # Build docker compose exec command for remote
+      compose_cmd = remote_docker_compose_exec(
+        remote_docker_compose_service,
+        util,
+        no_tty: true
+      )
+      return compose_cmd
+    end
+    mongodb_bin_path(util)
+  end
 
   def mongodb_dump_remote
     puts "backup remote DB via ssh"
@@ -65,7 +95,7 @@ module Cloner::MongoDB
       else
         username, password = mongodb_r_conf['username'], mongodb_r_conf['password']
       end
-      dump = "mongodump -u #{e username} -p #{e password} -d #{e mongodb_r_conf['database']} --authenticationDatabase #{e mongodb_r_conf['database']} -o #{e remote_dump_path} #{mongodb_dump_extra}"
+      dump = "#{mongodb_remote_bin_path 'mongodump'} -u #{e username} -p #{e password} -d #{e mongodb_r_conf['database']} --authenticationDatabase #{e mongodb_r_conf['database']} -o #{e remote_dump_path} #{mongodb_dump_extra}"
       puts dump if verbose?
       ret = ssh_exec!(ssh, dump)
       check_ssh_err(ret)
@@ -74,7 +104,7 @@ module Cloner::MongoDB
 
   def mongodb_dump_restore
     puts "restoring DB"
-    restore = "mongorestore --drop -d #{e mongodb_to} #{mongodb_local_auth} #{e mongodb_path}"
+    restore = "#{mongodb_local_bin_path 'mongorestore'} --drop -d #{e mongodb_to} #{mongodb_local_auth} #{e mongodb_path}"
     puts restore if verbose?
     pipe = IO.popen(restore)
     while (line = pipe.gets)