wordmove-ng 6.0.0

data/lib/wordmove/deployer/ssh.rb

@@ -0,0 +1,339 @@
+require 'pathname'
+
+module Wordmove
+  module Deployer
+    # Syncs files with rsync and the database with mysqldump/mysql over SSH.
+    #
+    # Database sync follows the same shape in both directions: dump the
+    # source, import the dump on the target, then run `wp search-replace` on
+    # the target for vhost and wordpress_path. The source database is only
+    # ever read, and wp-cli is required on whichever side is the target
+    # (remote for push, local for pull).
+    class SSH < Base
+      attr_reader :local_dump_path,
+                  :local_backup_path,
+                  :local_gzipped_dump_path,
+                  :local_gzipped_backup_path
+
+      def initialize(environment, options)
+        super
+
+        @runner = SshRunner.new(remote_options[:ssh])
+        @copier = Photocopier::SSH.new(photocopier_options).tap { |c| c.logger = logger }
+
+        @local_dump_path = local_wp_content_dir.path("dump.sql")
+        @local_backup_path = local_wp_content_dir.path("local-backup-#{Time.now.to_i}.sql")
+        @local_gzipped_dump_path = local_dump_path + '.gz'
+        @local_gzipped_backup_path = local_wp_content_dir
+                                     .path("#{environment}-backup-#{Time.now.to_i}.sql.gz")
+      end
+
+      private
+
+      # Photocopier deletes :gateway and :rsync_options from the hash it is given
+      # and we append --dry-run when simulating, so hand it a copy rather than the
+      # hash shared with the movefile options.
+      def photocopier_options
+        ssh_options = remote_options[:ssh].dup
+        return ssh_options unless simulate?
+
+        ssh_options[:rsync_options] = [ssh_options[:rsync_options], '--dry-run'].compact.join(' ')
+        ssh_options
+      end
+
+      def push_db
+        super
+
+        return true if simulate?
+
+        check_push_db_prerequisites!
+        backup_remote_db!
+        adapt_local_db!
+        after_push_cleanup!
+      end
+
+      def pull_db
+        super
+
+        return true if simulate?
+
+        check_pull_db_prerequisites!
+        backup_local_db!
+        adapt_remote_db!
+        after_pull_cleanup!
+      end
+
+      # Both checks run before any backup or dump so a missing dependency has
+      # no side effects.
+      def check_push_db_prerequisites!
+        check_prerequisites!(local: Prerequisites::DB_SOURCE, remote: Prerequisites::DB_TARGET)
+      end
+
+      def check_pull_db_prerequisites!
+        check_prerequisites!(local: Prerequisites::DB_TARGET, remote: Prerequisites::DB_SOURCE)
+      end
+
+      def check_prerequisites!(local:, remote:)
+        logger.task_step true, "checking prerequisites"
+        warn_about_prefix_collisions
+        missing_local = Prerequisites.missing_locally(local)
+        missing_remote = Prerequisites.missing_remotely(@runner, remote)
+        return true if missing_local.empty? && missing_remote.empty?
+
+        problems = []
+        problems << "locally: #{Prerequisites.describe(missing_local)}" if missing_local.any?
+        if missing_remote.any?
+          problems << "on \"#{environment}\": #{Prerequisites.describe(missing_remote)}"
+        end
+        raise UnmetPeerDependencyError,
+              "Missing programs required for the database sync (#{problems.join('; ')}). " \
+              "Install them, or make sure they are in the login shell $PATH."
+      end
+
+      def warn_about_prefix_collisions
+        Wordmove::Movefile.new(options[:config]).prefix_collisions(environment).each do |short, long|
+          logger.warn "\"#{short}\" is a prefix of \"#{long}\": the search-replace of the " \
+                      "shorter value will also rewrite the longer one. Check the result."
+        end
+      end
+
+      def backup_remote_db!
+        download_remote_db(local_gzipped_backup_path)
+      end
+
+      def adapt_local_db!
+        save_local_db(local_dump_path)
+        normalize_collations!(local_dump_path)
+        run compress_command(local_dump_path)
+        with_maintenance_mode(remote: true) do
+          import_remote_dump(local_gzipped_dump_path)
+          adapt_remote_db_after_import!
+        end
+      end
+
+      def after_push_cleanup!
+        local_delete(local_gzipped_dump_path)
+      end
+
+      def backup_local_db!
+        save_local_db(local_backup_path)
+        run compress_command(local_backup_path)
+      end
+
+      def adapt_remote_db!
+        download_remote_db(local_gzipped_dump_path)
+        run uncompress_command(local_gzipped_dump_path)
+        normalize_collations!(local_dump_path)
+        with_maintenance_mode(remote: false) do
+          run mysql_import_command(local_dump_path, local_options[:database])
+          run_wpcli_search_replace(remote_options, local_options, :vhost)
+          run_wpcli_search_replace(remote_options, local_options, :wordpress_path)
+        end
+      end
+
+      # Wraps the block in `wp maintenance-mode activate` / `deactivate` on the
+      # target when global.maintenance_mode (or WORDMOVE_MAINTENANCE_MODE) is
+      # set, so visitors see a maintenance page instead of a half-adapted site.
+      # Deactivation always runs, even when the block raises.
+      def with_maintenance_mode(remote:)
+        return yield unless maintenance_mode?
+
+        side = remote ? remote_options : local_options
+        runner = remote ? method(:remote_run) : method(:run)
+        runner.call(SqlAdapter::Wpcli.maintenance_mode_command(:activate, side[:wordpress_path]))
+        begin
+          yield
+        ensure
+          runner.call(SqlAdapter::Wpcli.maintenance_mode_command(:deactivate,
+                                                                 side[:wordpress_path]))
+        end
+      end
+
+      def maintenance_mode?
+        env = ENV.fetch('WORDMOVE_MAINTENANCE_MODE', nil)
+        return %w[1 true yes on].include?(env.strip.downcase) unless env.nil? || env.strip.empty?
+
+        options.dig(:global, :maintenance_mode) == true
+      end
+
+      def after_pull_cleanup!
+        local_delete(local_dump_path)
+      end
+
+      def adapt_remote_db_after_import!
+        remote_run_wpcli_search_replace(local_options, remote_options, :vhost)
+        remote_run_wpcli_search_replace(local_options, remote_options, :wordpress_path)
+      rescue ShellCommandError => e
+        logger.error "The remote database was imported but could not be adapted: #{e.message}"
+        logger.error "The remote site may now reference the local vhost or path. " \
+                     "A backup of the remote database taken before the import is at: " \
+                     "#{local_gzipped_backup_path}"
+        raise
+      end
+
+      def wpcli_search_replace(from, to, config_key, path:, remote:)
+        return if options[:no_adapt]
+
+        logger.task_step !remote, "adapt dump for #{config_key}"
+        return if simulate?
+
+        SqlAdapter::Wpcli.new(from, to, config_key, path, remote: remote).command
+      end
+
+      def run_wpcli_search_replace(from, to, config_key)
+        command = wpcli_search_replace(
+          from, to, config_key, path: local_options[:wordpress_path], remote: false
+        )
+        run(command) if command
+      end
+
+      def remote_run_wpcli_search_replace(from, to, config_key)
+        command = wpcli_search_replace(
+          from, to, config_key, path: remote_options[:wordpress_path], remote: true
+        )
+        remote_run(command) if command
+      end
+
+      # Directory transfers go through rsync (Photocopier), which honours the
+      # --dry-run option set in #initialize when simulating.
+      %w[get_directory put_directory].each do |command|
+        define_method "remote_#{command}" do |*args|
+          logger.task_step false, "#{command}: #{args.join(' ')}"
+          @copier.send(command, *args)
+        end
+      end
+
+      # Single file transfers and deletes go through the system scp/ssh binaries.
+      def remote_get(remote_path, local_path)
+        logger.task_step false, "get: #{remote_path} #{local_path}"
+        return true if simulate?
+
+        ensure_success!(@runner.get(remote_path, local_path), "get #{remote_path}")
+      end
+
+      def remote_put(local_path, remote_path)
+        logger.task_step false, "put: #{local_path} #{remote_path}"
+        return true if simulate?
+
+        ensure_success!(@runner.put(local_path, remote_path), "put #{remote_path}")
+      end
+
+      def remote_delete(remote_path)
+        logger.task_step false, "delete: #{remote_path}"
+        return true if simulate?
+
+        ensure_success!(@runner.delete(remote_path), "delete #{remote_path}")
+      end
+
+      def remote_run(command)
+        logger.task_step false, command
+        return true if simulate?
+
+        ensure_success!(@runner.run(command), command)
+      end
+
+      def ensure_success!(result, description)
+        _stdout, stderr, exit_code = result
+        return true if exit_code.zero?
+
+        raise(
+          ShellCommandError,
+          "Error code #{exit_code} returned by command \"#{description}\": #{stderr}"
+        )
+      end
+
+      def download_remote_db(local_gizipped_dump_path)
+        remote_dump_path = remote_wp_content_dir.path("dump.sql")
+        # dump remote db into file
+        remote_run mysql_dump_command(remote_options[:database], remote_dump_path)
+        remote_run compress_command(remote_dump_path)
+        remote_dump_path += '.gz'
+        # download remote dump
+        remote_get(remote_dump_path, local_gizipped_dump_path)
+        remote_delete(remote_dump_path)
+      end
+
+      def import_remote_dump(local_gizipped_dump_path)
+        remote_dump_path = remote_wp_content_dir.path("dump.sql")
+        remote_gizipped_dump_path = remote_dump_path + '.gz'
+
+        remote_put(local_gizipped_dump_path, remote_gizipped_dump_path)
+        remote_run uncompress_command(remote_gizipped_dump_path)
+        remote_run mysql_import_command(remote_dump_path, remote_options[:database])
+        remote_delete(remote_dump_path)
+      end
+
+      %w[uploads themes plugins mu_plugins languages].each do |task|
+        define_method "push_#{task}" do
+          logger.task "Pushing #{task.titleize}"
+          local_path = local_options[:wordpress_path]
+          remote_path = remote_options[:wordpress_path]
+
+          remote_put_directory(local_path, remote_path,
+                               push_exclude_paths(task), push_inlcude_paths(task))
+        end
+
+        define_method "pull_#{task}" do
+          logger.task "Pulling #{task.titleize}"
+          local_path = local_options[:wordpress_path]
+          remote_path = remote_options[:wordpress_path]
+
+          remote_get_directory(remote_path, local_path,
+                               pull_exclude_paths(task), pull_include_paths(task))
+        end
+      end
+
+      def push_inlcude_paths(task)
+        Pathname.new(send(:"local_#{task}_dir").relative_path)
+                .ascend
+                .each_with_object([]) do |directory, array|
+                  path = directory.to_path
+                  path.prepend('/') unless path.match? %r{^/}
+                  path.concat('/') unless path.match? %r{/$}
+                  array << path
+                end
+      end
+
+      def push_exclude_paths(task)
+        Pathname.new(send(:"local_#{task}_dir").relative_path)
+                .dirname
+                .ascend
+                .each_with_object([]) do |directory, array|
+                  path = directory.to_path
+                  path.prepend('/') unless path.match? %r{^/}
+                  path.concat('/') unless path.match? %r{/$}
+                  path.concat('*')
+                  array << path
+                end
+                .concat(paths_to_exclude)
+                .push('/*')
+      end
+
+      def pull_include_paths(task)
+        Pathname.new(send(:"remote_#{task}_dir").relative_path)
+                .ascend
+                .each_with_object([]) do |directory, array|
+                  path = directory.to_path
+                  path.prepend('/') unless path.match? %r{^/}
+                  path.concat('/') unless path.match? %r{/$}
+                  array << path
+                end
+      end
+
+      def pull_exclude_paths(task)
+        Pathname.new(send(:"remote_#{task}_dir").relative_path)
+                .dirname
+                .ascend
+                .each_with_object([]) do |directory, array|
+                  path = directory.to_path
+                  path.prepend('/') unless path.match? %r{^/}
+                  path.concat('/') unless path.match? %r{/$}
+                  path.concat('*')
+                  array << path
+                end
+                .concat(paths_to_exclude)
+                .push('/*')
+      end
+    end
+  end
+end

data/lib/wordmove/doctor/movefile.rb

@@ -0,0 +1,112 @@
+module Wordmove
+  class Doctor
+    class Movefile
+      MANDATORY_SECTIONS = %i[local].freeze
+      OPTIONAL_SECTIONS = %i[global].freeze
+      attr_reader :movefile, :contents, :root_keys
+
+      def initialize(name = nil, dir = '.')
+        @movefile = Wordmove::Movefile.new(name, dir)
+
+        begin
+          @contents = movefile.fetch
+          @root_keys = contents.keys
+        rescue Psych::SyntaxError
+          movefile.logger.error "Your movefile is not parsable due to a syntax error" \
+                                "so we can't continue to validate it."
+          movefile.logger.debug "You could try to use https://yamlvalidator.com/ to" \
+                                "get a clue about the problem."
+        end
+      end
+
+      def validate!
+        return false unless root_keys
+
+        MANDATORY_SECTIONS.each do |key|
+          movefile.logger.task "Validating movefile section: #{key}"
+          validate_mandatory_section(key)
+        end
+
+        OPTIONAL_SECTIONS.each do |key|
+          next unless root_keys.delete(key)
+
+          movefile.logger.task "Validating movefile section: #{key}"
+          validate_section(key)
+        end
+
+        root_keys.each do |remote|
+          movefile.logger.task "Validating movefile section: #{remote}"
+          validate_remote_section(remote)
+        end
+      end
+
+      private
+
+      def validate_section(key)
+        validator = validator_for(key)
+
+        errors = validator.validate(contents[key].deep_stringify_keys)
+
+        if errors && errors.empty?
+          movefile.logger.success "Formal validation passed"
+
+          return true
+        end
+
+        errors.each do |e|
+          movefile.logger.error "[#{e.path}] #{e.message}"
+        end
+      end
+
+      def validate_mandatory_section(key)
+        return false unless root_keys.delete(key) do
+          movefile.logger.error "#{key} section not present"
+
+          false
+        end
+
+        validate_section(key)
+      end
+
+      def validate_remote_section(key)
+        return false unless validate_protocol_presence(contents[key].keys)
+
+        validate_prefix_collisions(key)
+        validate_section(key)
+      end
+
+      def validate_prefix_collisions(key)
+        movefile.prefix_collisions(key).each do |short, long|
+          movefile.logger.error "\"#{short}\" is a prefix of \"#{long}\": wp search-replace " \
+                                "would rewrite the longer value too. Use distinct vhosts " \
+                                "and wordpress_paths for local and #{key}."
+        end
+      end
+
+      def validate_protocol_presence(keys)
+        if keys.include?(:ftp)
+          movefile.logger.error "This remote is configured with `ftp`, but FTP support was " \
+                                "removed in wordmove-ng 6.0. Switch it to `ssh`."
+          return false
+        end
+        return true if keys.include?(:ssh)
+
+        movefile.logger.error "This remote has no ssh protocol defined"
+
+        false
+      end
+
+      def validator_for(key)
+        suffix = if (MANDATORY_SECTIONS + OPTIONAL_SECTIONS).include? key
+                   key
+                 else
+                   'remote'
+                 end
+
+        schema = Kwalify::Yaml.load_file("#{__dir__}/../assets/wordmove_schema_#{suffix}.yml")
+
+        Kwalify::Validator.new(schema)
+      end
+    end
+  end
+end

data/lib/wordmove/doctor/mysql.rb

@@ -0,0 +1,136 @@
+module Wordmove
+  class Doctor
+    class Mysql
+      attr_reader :config, :logger
+
+      def initialize(movefile_name = nil, movefile_dir = '.')
+        @logger = Logger.new(STDOUT).tap { |l| l.level = Logger::INFO }
+        begin
+          @config = Wordmove::Movefile.new(movefile_name, movefile_dir).fetch[:local][:database]
+        rescue Psych::SyntaxError
+          nil
+        end
+      end
+
+      def check!
+        logger.task "Checking local database commands and connection"
+
+        return logger.error "Can't connect to mysql using your movefile.yml" if config.nil?
+
+        mysql_client_doctor
+        mysqldump_doctor
+        mysql_server_doctor
+        mysql_database_doctor
+      end
+
+      private
+
+      def mysql_client_doctor
+        if mysql_client_in_path?
+          logger.success "`mysql`/`mariadb` command is in $PATH"
+        else
+          logger.error "`mysql`/`mariadb` command is not in $PATH"
+        end
+      end
+
+      def mysqldump_doctor
+        if mysqldump_in_path?
+          logger.success "`mysqldump`/`mariadb-dump` command is in $PATH"
+        else
+          logger.error "`mysqldump`/`mariadb-dump` command is not in $PATH"
+        end
+      end
+
+      def mysql_server_doctor
+        command = mysql_command
+
+        if system(command, out: File::NULL, err: File::NULL)
+          logger.success "Successfully connected to the MySQL server"
+        else
+          logger.error <<-LONG
+  We can't connect to the MySQL server using credentials
+                specified in the Movefile. Double check them or try
+                to debug your system configuration.
+
+                The command used to test was:
+
+                #{command}
+          LONG
+        end
+      end
+
+      def mysql_database_doctor
+        command = mysql_command(database: config[:name])
+
+        if system(command, out: File::NULL, err: File::NULL)
+          logger.success "Successfully connected to the database"
+        else
+          logger.error <<-LONG
+  We can't connect to the database using credentials
+                specified in the Movefile, or the database does not
+                exists. Double check them or try to debug your
+                system configuration.
+
+                The command used to test was:
+
+                #{command}
+          LONG
+        end
+      end
+
+      def mysql_command(database: nil)
+        mysql_options = config[:mysql_options]
+        command = [mysql_client_binary]
+        command << "--host=#{Shellwords.escape(config[:host])}" if config[:host].present?
+        command << "--port=#{Shellwords.escape(config[:port])}" if config[:port].present?
+        if (socket = mysql_socket_option)
+          command << "--socket=#{Shellwords.escape(socket)}"
+        end
+        command << "--user=#{Shellwords.escape(config[:user])}" if config[:user].present?
+        if config[:password].present?
+          command << "--password=#{Shellwords.escape(config[:password])}"
+        end
+        command << Shellwords.split(mysql_options) if mysql_options.present?
+        command << database if database.present?
+        command << "-e'QUIT'"
+        command.join(" ")
+      end
+
+      def mysql_client_binary
+        '$(command -v mariadb >/dev/null 2>&1 && echo mariadb || echo mysql)'
+      end
+
+      def mysql_client_in_path?
+        system("which mysql", out: File::NULL) || system("which mariadb", out: File::NULL)
+      end
+
+      def mysqldump_in_path?
+        system("which mysqldump", out: File::NULL) || system("which mariadb-dump", out: File::NULL)
+      end
+
+      def mysql_socket_option
+        return if mysql_option_includes_socket?(config[:mysql_options])
+
+        config[:socket].presence || mysql_socket_from_options(config[:mysqldump_options])
+      end
+
+      def mysql_option_includes_socket?(option_string)
+        mysql_socket_from_options(option_string).present?
+      end
+
+      def mysql_socket_from_options(option_string)
+        return if option_string.to_s.empty?
+
+        args = Shellwords.split(option_string)
+        socket_index = args.index('--socket')
+        return args[socket_index + 1] if socket_index && args[socket_index + 1]
+
+        args.each do |arg|
+          return Regexp.last_match(1) if arg.match(/\A--socket=(.+)\z/)
+        end
+
+        nil
+      end
+    end
+  end
+end

data/lib/wordmove/doctor/rsync.rb

@@ -0,0 +1,27 @@
+module Wordmove
+  class Doctor
+    class Rsync
+      attr_reader :logger
+
+      def initialize
+        @logger = Logger.new(STDOUT).tap { |l| l.level = Logger::INFO }
+      end
+
+      def check!
+        logger.task "Checking rsync"
+
+        version_output = `rsync --version | head -n1 2>&1`
+
+        if version_output.downcase.include?("openrsync")
+          protocol_version = version_output[/protocol version (\d+)/i, 1]
+          logger.success "openrsync detected (protocol version #{protocol_version || 'unknown'})"
+        elsif (match = /\d+\.\d+\.\d+/.match(version_output))
+          logger.success "rsync is installed at version #{match[0]}"
+        else
+          logger.error "rsync not found or the version could not be detected. " \
+                       "Output was: #{version_output.strip}"
+        end
+      end
+    end
+  end
+end