skadategems-dev 0.0.9

data/integration/lib/remote/download_file_spec.rb

@@ -0,0 +1,136 @@
+require File.expand_path('../../spec_helper', __dir__)
+require 'skadategems/dev/skadate'
+require 'fileutils'
+require 'digest/md5'
+
+module SkadateGems::Dev
+  describe Remote::File do
+    let(:skadate) { Skadate.new(skadate_local_dir) }
+
+    let(:big_file_path) do
+      ENV['SKADATE_BIG_FILE_PATH'] || File.join(skadate_source_dir, 'install/skadate9.sql')
+    end
+
+    let(:big_file_name) { File.basename(big_file_path) }
+
+    let(:remote_file) do
+      Remote::File.new(skadate.remote, big_file_name, File.size(big_file_path))
+    end
+
+    before(:each) do
+      remote_install
+
+      symlink = File.join(skadate_remote_dir, big_file_name)
+      FileUtils.ln_s(big_file_path, symlink, verbose: verbose?)
+
+      FileUtils.cd(skadate_local_dir, verbose: verbose?)
+    end
+
+    describe '#request' do
+
+      it 'yields `http response` that allows us to #read_body as a stream' do
+        segments_count = 0
+
+        remote_file.request do |response|
+          response.read_body { |_| segments_count += 1 }
+        end
+
+        puts "[segments count = #{segments_count}]" if verbose?
+        expect(segments_count).to be > 100
+      end
+
+      context 'when :md5_checksum is given' do
+
+        context "and remote file's checksum is the same" do
+          let(:checksum) { Digest::MD5.file(big_file_path).hexdigest }
+
+          it 'responds with [304 "Not Modified"] and empty body' do
+            remote_file.request(md5_checksum: checksum) do |response|
+              expect(response).to be_a Net::HTTPNotModified
+              response.read_body { |_| raise 'should not have been called!' }
+            end
+          end
+        end
+
+        context "but remote file's checksum is different" do
+          let(:checksum) { 'a' * 32 }
+
+          it 'responds with [200 "OK"] and file content as a stream' do
+            segments_count = 0
+
+            remote_file.request(md5_checksum: checksum) do |response|
+              expect(response).to be_a Net::HTTPOK
+              response.read_body { |_| segments_count += 1 }
+            end
+
+            puts "[segments count = #{segments_count}]" if verbose?
+            expect(segments_count).to be > 100
+          end
+        end
+
+      end
+
+    end
+
+    describe '#save_as(filename)' do
+      let(:local_file_path) { File.join(skadate_local_dir, big_file_name) }
+
+      context 'when there is no local file with a given filename' do
+        it 'downloads and saves a remote file' do
+          return_value = nil
+
+          expect {
+            return_value = remote_file.save_as(local_file_path)
+          }.to change { File.exists?(local_file_path) }.to true
+
+          expect(return_value).to be_true
+
+          expect(FileUtils.identical?(big_file_path, local_file_path)).to be_true
+        end
+      end
+
+      context 'when a given file is already exists' do
+
+        context 'and files are the same' do
+          before(:each) do
+            FileUtils.cp(big_file_path, local_file_path, verbose: verbose?)
+            expect(FileUtils.identical?(big_file_path, local_file_path)).to be_true
+          end
+
+          it "doesn't download a remote file" do
+            FileUtils.chmod('u-w', local_file_path, verbose: verbose?)
+            expect(File.writable?(local_file_path)).to be_false
+
+            return_value = nil
+
+            expect {
+              return_value = remote_file.save_as(local_file_path)
+            }.to_not change { File.writable?(local_file_path) }
+
+            expect(return_value).to eq false
+          end
+        end
+
+        context 'but the files are different' do
+          before(:each) do
+            File.write(local_file_path, 'Oh Noo!')
+          end
+
+          it 'downloads and rewrites a local file' do
+            return_value = nil
+
+            expect {
+              return_value = remote_file.save_as(local_file_path)
+            }.to change {
+              FileUtils.identical?(local_file_path, big_file_path)
+            }.to true
+
+            expect(return_value).to be_true
+          end
+        end
+
+      end
+    end
+
+  end
+end

data/integration/spec_helper.rb

@@ -0,0 +1,101 @@
+# integration/spec_helper.rb
+
+require File.expand_path('../spec/spec_helper', __dir__)
+require 'skadategems/dev/cli/skadate'
+require 'fileutils'
+
+RSpec.configure do |config|
+
+  config.before(:each) do
+    # cleanup sandbox directories
+    [skadate_local_dir, skadate_remote_dir].each do |entry|
+      if entry.is_a?(String) && (entry['tmp'] || entry['temp']) && entry['skadate']
+        FileUtils.rm_r Dir.glob("#{entry}/{*,.dev,.ht*}"), :verbose => verbose?
+      else
+        raise <<-ERRORMSG
+Error: bad tmp dir entry "#{path}".
+Explanation: by security reason tmp directories must contain 'tmp' and 'skadate' keywords in their pathname.
+        ERRORMSG
+      end
+    end
+  end
+
+  def env_var!(name)
+    ENV[name] or raise "`#{name}` is not defined"
+  end
+
+  def verbose?
+    (verbose = ENV['VERBOSE']) && verbose != '' && verbose.upcase != 'FALSE'
+  end
+
+  def skadate_tmp_dir
+    @skadate_tmp_dir ||= File.join(File.symlink?('/tmp') ? "/#{File.readlink('/tmp')}" : '/tmp', 'skadate')
+  end
+
+  def skadate_source_dir
+    @skadate_source_dir ||= env_var! 'SKADATE_SOURCE_DIR'
+  end
+
+  def skadate_local_dir
+    @skadate_local_dir ||= ENV['SKADATE_LOCAL_DIR'] || File.join(skadate_tmp_dir, 'local')
+  end
+
+  def skadate_remote_dir
+    @skadate_remote_dir ||= ENV['SKADATE_REMOTE_DIR'] || File.join(skadate_tmp_dir, 'remote')
+  end
+
+  def skadate_local_host
+    ENV['SKADATE_LOCAL_HOST'] || 'skadate-local-test'
+  end
+
+  def skadate_remote_host
+    ENV['SKADATE_REMOTE_HOST'] || 'skadate-remote-test'
+  end
+
+  def remote_install(options = {})
+    FileUtils.cd(skadate_remote_dir, :verbose => verbose?) do
+      FileUtils.mkdir('internals', :verbose => verbose?)
+
+      puts '[...] > internals/config.php' if verbose?
+      IO.write('internals/config.php', <<-CONFIGPHP)
+<?php
+define('DIR_SITE_ROOT', '#{skadate_remote_dir}/');
+define('SITE_URL', 'http://#{skadate_remote_host}/');
+      CONFIGPHP
+    end
+
+    FileUtils.cd(skadate_local_dir, :verbose => verbose?) do
+      puts "skadate init -r http://#{skadate_remote_host}/" if verbose?
+      SkadateGems::Dev::CLI::Skadate.start ['init', '-r', "http://#{skadate_remote_host}/"]
+      FileUtils.cp('exec.php', skadate_remote_dir, :verbose => verbose?)
+    end
+  end
+
+  def local_install(options = {})
+  end
+
+  def source_copy(source_file_paths, destination_root = :local, options = {})
+    destination_root = skadate_local_dir if destination_root == :local
+    destination_root = skadate_remote_dir if destination_root == :remote
+
+    [*source_file_paths].each do |entry|
+      src = File.join(skadate_source_dir, entry)
+      dest = File.join(destination_root, entry)
+      FileUtils.mkdir_p File.dirname(dest), :verbose => verbose?
+      FileUtils.cp(src, dest, options.merge(:verbose => verbose?))
+    end
+  end
+
+  # @param bytes [Fixnum] value in bytes
+  # @return [String] human readable format
+  def human_readable_bytes(bytes)
+    units = %W(B KiB MiB GiB TiB)
+
+    size, unit = units.reduce(bytes.to_f) do |(fsize, _), type|
+      fsize > 512 ? [fsize / 1024, type] : (break [fsize, type])
+    end
+
+    "#{size > 9 || size.modulo(1) < 0.1 ? '%d' : '%.1f'} %s" % [size, unit]
+  end
+
+end

data/lib/skadategems/dev/bundle.rb

@@ -0,0 +1,115 @@
+require 'rubygems'
+require 'rubygems/package'
+require 'zlib'
+require 'fileutils'
+
+module SkadateGems
+  module Dev
+
+    class Bundle
+
+      attr_accessor :layout_theme
+
+      # Bundler instance initializer.
+      #
+      # @param skadate [Skadate]
+      def initialize(skadate)
+        @source     = skadate.source
+        @internal_c = @source.internal_c
+        @external_c = @source.external_c
+        @userfiles  = @source.userfiles
+        @themes_dir = @source.filename('layout/themes/')
+      end
+
+      # Filter files by their filename.
+      #
+      # @param filename [String] absolute path to a file
+      # @return [boolean] true if given filename should be bundled
+      def should_include?(filename)
+        return false if filename.start_with?(@internal_c, @external_c, @userfiles) ||
+            filename.end_with?('.tar', '.gz', '.zip', '.swp', '~', 'Thumbs.db', 'desktop.ini')
+
+        if filename.start_with?(@themes_dir) && layout_theme &&
+          !filename.start_with?(File.join(@themes_dir, layout_theme))
+          return false
+        end
+
+        true
+      end
+
+      # Create a tar file in memory.
+      #
+      # @return [StringIO] whose underlying String is the contents of the tar file
+      def tar
+        return @tar if @tar
+
+        string_io = StringIO.new('')
+        Gem::Package::TarWriter.new(string_io) do |tar|
+
+          htaccess = File.join(@source.root, '.htaccess')
+          if File.exists?(htaccess)
+            tar.add_file '.htaccess', 0644 do |tar_file|
+              File.open(htaccess, 'rb') { |file| tar_file.write(file.read) }
+            end
+          end
+
+          Dir[File.join(@source.root, '**/*')].each do |filename|
+            next unless should_include?(filename)
+
+            if File.directory?(filename)
+              tar.mkdir (relative_path filename), 0755
+
+              htaccess = File.join(filename, '.htaccess')
+              if File.exists?(htaccess)
+                filename = htaccess # iteration continues...
+              else
+                next
+              end
+            end
+
+            tar.add_file (relative_path filename), 0644 do |tar_file|
+              File.open(filename, 'rb') { |file| tar_file.write(file.read) }
+            end
+          end
+
+          [ (int_c = relative_path @internal_c),
+            "#{int_c}/cache",
+            "#{int_c}/components",
+            "#{int_c}/forms",
+            "#{int_c}/lang",
+            relative_path(@external_c),
+            relative_path(@userfiles),
+          ].each { |dir| tar.mkdir dir, 0777 }
+        end
+
+        string_io.rewind
+        @tar = string_io
+      end
+
+      # gzips the underlying string in the given `StringIO`,
+      # returning a new `StringIO` representing the compressed file.
+      #
+      # @param filename [String] optional output filename
+      # @return [StringIO|Fixnum]
+      def gzip(filename = nil)
+        gz = StringIO.new('')
+        z = Zlib::GzipWriter.new(gz)
+        z.write(tar.string)
+        z.close # this is necessary!
+
+        if filename
+          File.binwrite(filename, gz.string)
+        else
+          StringIO.new(gz.string)
+        end
+      end
+
+    private
+      def relative_path(filename)
+        filename.sub(/^#{Regexp.escape(@source.root)}\/?/, '')
+      end
+
+    end
+
+  end
+end

data/lib/skadategems/dev/cli/remote.rb

@@ -0,0 +1,364 @@
+require 'time'
+require 'thor'
+require 'json'
+require 'skadategems/dev/skadate'
+require 'skadategems/dev/remote/clone_logger'
+require 'ruby-progressbar'
+
+module SkadateGems::Dev
+  module CLI
+
+    class Remote < Thor
+      namespace 'remote'
+
+      desc 'ping', 'Send ping request to a remote application'
+      long_desc <<-LONGDESC
+Send ping request to a remote server's `exec.php` script. \x5
+\x5
+\x5 Example usage: \x5
+\x5
+\x5  $> skadate remote ping
+\x5
+\x5
+      LONGDESC
+
+      def ping
+        accessor = current.remote.accessor
+        script = ExecPHP::ScriptBatch.new { |s| s << 'echo "pong!";' }
+
+        say '=> '
+        say accessor.execphp_uri, :cyan
+
+        accessor.exec(script) do |res|
+          say '#> '
+          say "#{res.code} [#{res.message}]", (res.code == '200' ? :green : :red)
+
+          if res.body.empty?
+            say 'Error: empty response body', :red
+          elsif res.body != 'pong!'
+            say 'Error: wrong response body:', :red
+            if res.body.length > 1000
+              say res.body[0..984], nil, false
+              say ' ... (continued)', :yellow
+            else
+              say '"' + res.body + '"'
+            end
+          else
+            say '>> '
+            say res.body, :white
+          end
+        end
+      end
+
+      desc 'exec FILENAME', 'Execute php scripts remotely'
+      long_desc <<-LONGDESC
+Send a given php file to a remote application server for execution.
+
+Example usage:
+\x5
+\x5  $> skadate remote exec path/to/my_script.php
+\x5
+\x5
+      LONGDESC
+
+      def exec(filename)
+        puts current.remote.exec { |batch| batch.include_file(filename) }.body
+      end
+
+      desc 'config ID [VALUE]', 'Manage remote application configuration'
+
+      option :type, :type => :string,
+                    :desc => 'Explicitly specify config value type'
+
+      def config(key_or_id, value = nil)
+        key_or_id = key_or_id.to_i if numeric? key_or_id
+        config = current.remote.configs[key_or_id]
+
+        say '>> ', (:yellow unless value.nil?), false
+        say "##{config.id} ", :white, false
+        say '| '
+        say config.name, :cyan, false
+        say ': '
+        say config.value.to_json, (color_for config.value), value.nil?
+
+        unless value.nil?
+          value =
+            case options[:type]
+              when nil
+                if value == 'true'
+                  true
+                elsif value == 'false'
+                  false
+                elsif numeric? value
+                  value.to_i
+                else
+                  value
+                end
+              when 'str'
+              when 'string'
+                value
+              when 'int'
+              when 'integer'
+              when 'number'
+                value.to_i
+              when 'bool'
+              when 'boolean'
+                value != 'false' && value != '0'
+              when 'json'
+                JSON.parse(value)
+              else
+                raise "unrecognized config value type '#{options[:type]}'"
+            end
+
+          say ' => ', :white, false
+          say value.to_json, (color_for value)
+
+          say '>> '
+          if yes?('perform this update? [yn]')
+            config.value = value
+
+            case config.update_status
+              when :updated
+                say '[updated]', :green
+              when :not_modified
+                say '[not modified]', :yellow
+              else
+                say '[error]', :red
+            end
+          else
+            say '[cancelled]', :red
+          end
+        end
+      end
+
+      desc 'clone', 'Clone production-running application'
+      long_desc <<-LONGDESC
+Clone production-running application locally for development purposes.
+
+To download remote application source files:
+\x5
+\x5  $> skadate remote clone --sources \x5
+\x5
+
+To get a remote application database dump:
+\x5
+\x5  $> skadate remote clone --database \x5
+\x5
+
+Or if you want to make a full clone of a running application including all the data and user uploaded files:
+\x5
+\x5  $> skadate remote clone --sources --database --userfiles \x5
+\x5
+
+Make a note that by default `remote clone --sources` copies only one `layout/theme` that is currently active on a remote server.
+If you want to get another theme specify it using the `--layout-theme` switch as following:
+\x5
+\x5  $> skadate remote clone --layout-theme stencil \x5
+\x5
+\x5
+      LONGDESC
+
+      option :sources, :type => :boolean,
+                       :desc => 'Copy application source files'
+
+      option :schema,  :type => :boolean,
+                       :desc => 'Copy database schema'
+
+      option :configs, :type => :boolean,
+                       :desc => 'Copy application-related configs data'
+
+      option :database, :type => :boolean,
+                        :desc => 'Copy the whole application database'
+
+      option :userfiles, :type => :boolean,
+                         :desc => 'Copy `userfiles` directory contents'
+
+      option :layout_theme, :type => :string,
+                            :banner => 'NAME',
+                            :desc => 'Force layout theme'
+
+      IGNORED_DIR_PATHS = %w[
+        /$external_c /$internal_c /$userfiles /$userfiles/tmp
+         /external_c  /internal_c  /userfiles  /userfiles/tmp
+        /cgi-bin
+        /layout/themes
+        /m/application/cache/smarty_compile
+      ]
+
+      IGNORED_FILE_EXTENSIONS = %w[.tar .bz2 .gz .zip .rar .iso]
+
+      # default values, rewritable by ENV['SKADATE_MAX_*_FILE_SIZE']
+      MAX_SOURCE_FILE_SIZE = 1024 * 1024 * 2 # ~2 megabytes
+      MAX_USER_FILE_SIZE   = 1024 * 1024 * 10 # ~10 megabytes
+
+      # show progressbar when downloading file size exceeds this value
+      SHOW_PROGRESS_FILE_SIZE = 1024 * 512 # ~512 kilobytes
+
+      def clone
+        if options[:schema] || options[:configs]
+          say <<-STDOUT, :red
+Remote clone options `--schema`, `--configs` are currently not implemented, please use `--database` for now.
+          STDOUT
+          return
+        end
+
+        if options.size == 0
+          say 'Error: `remote clone` options are required', :red
+          return help('clone')
+        end
+
+        remote = current.remote
+        layout_theme = options[:layout_theme]
+        @max_file_size = MAX_SOURCE_FILE_SIZE
+
+        dev_logs_dir = ::File.join(current.root_dir, '.dev', 'log')
+        FileUtils.mkdir_p(dev_logs_dir) unless ::File.exists?(dev_logs_dir)
+        log_filename = ::File.join(dev_logs_dir, "remote-clone[#{Time.now.to_i}].log")
+
+        logger.start(log_filename) do |log|
+          if options[:sources]
+            log.puts '>> Downloading application sources...', :yellow
+            download_source_dir(remote.root, log)
+
+            if layout_theme.nil?
+              log.puts('>> Getting active theme name...', :yellow)
+              config = remote.configs[1]
+              raise <<-ERRMSG unless config.name == 'theme'
+unexpected remote application config entry `#{config.name}`, (expected `theme` [config_id: 1])
+              ERRMSG
+              layout_theme = config.value
+            end
+          end
+
+          if layout_theme
+            log.puts ">> Downloading [/layout/themes/#{layout_theme}/*]...", :yellow
+            download_source_dir remote.directory("/layout/themes/#{layout_theme}"), log
+          end
+
+          if options[:database]
+            log.puts '>> Creating remote database dump...', :yellow
+            remote.create_mysql_dump do |remote_file|
+              local_filename = File.join(current.root_dir,
+                  "remote-db-clone.#{remote_file.basename =~ /\.tar\.gz$/ ? 'tar.gz' : 'sql'}")
+
+              if log.mute?
+                remote_file.save_as(local_filename)
+              else
+                progressbar = nil
+                remote_file.save_as(local_filename) do |chunk_size|
+                  if progressbar
+                    progressbar.progress += chunk_size
+                  else
+                    progressbar = create_progressbar(remote_file, log.padding, chunk_size)
+                  end
+                end
+              end
+
+              log.puts '>> Remote cache cleanup...', :yellow
+            end ? log.puts('>> Done!', :green) : log.puts('>> Warning: remote cache cleanup failed', :red)
+          end
+
+          if options[:userfiles]
+            log.puts '>> Detecting `$userfiles` directory...', :yellow
+            @max_file_size = MAX_USER_FILE_SIZE
+            download_source_dir(remote.userfiles, log)
+          end
+
+          log.puts '>> Completed!', :green
+        end
+      end
+
+    protected
+      def current
+        @current ||= SkadateGems::Dev::Skadate.new(Dir.pwd)
+      end
+
+      def numeric?(string)
+        string =~ /\A\d+\z/
+      end
+
+      def color_for(value)
+        if value.is_a? String
+          :green
+        elsif value.is_a? Fixnum
+          :blue
+        elsif value.is_a?(TrueClass) || value.is_a?(FalseClass)
+          :magenta
+        elsif value.is_a?(Array) || value.is_a?(Hash)
+          :yellow
+        else
+          :red
+        end
+      end
+
+      def logger
+        SkadateGems::Dev::Remote::CloneLogger
+      end
+
+      # @param directory [Remote::Directory] remote directory entry
+      # @param log [Remote::CloneLogger] logger instance
+      def download_source_dir(directory, log)
+        list = nil
+        log.entry(directory) { list = directory.list }
+        return unless list
+
+        log.padding += 1
+
+        begin
+          list.each do |entry|
+            if entry.directory?
+              if IGNORED_DIR_PATHS.include?(entry.path)
+                log.entry(entry, :skipped, 'skipped by pathname') { }
+              else
+                download_source_dir(entry, log)
+              end
+            else
+              if IGNORED_FILE_EXTENSIONS.include?(entry.extname)
+                log.entry(entry, :skipped, "skipped by (*#{entry.extname})") { }
+              elsif entry.size > @max_file_size
+                log.entry(entry, :skipped, 'skipped by filesize') { }
+              else
+                log.entry(entry) do
+                  local_filename = File.join(current.root_dir, entry.path)
+
+                  downloaded =
+                    if !log.mute? && entry.size > SHOW_PROGRESS_FILE_SIZE
+                      progressbar = nil
+                      entry.save_as(local_filename) do |chunk_size|
+                        if progressbar
+                          progressbar.progress += chunk_size
+                        else
+                          progressbar = create_progressbar(entry, log.padding, chunk_size)
+                        end
+                      end
+                    else
+                      entry.save_as(local_filename)
+                    end
+
+                  downloaded ? :downloaded : :unchanged
+                end
+              end
+            end
+          end
+        ensure
+          log.padding -= 1
+        end
+      end
+
+      def create_progressbar(entry, padding, starting_at = 0)
+        format =
+"#{'  ' * padding}~> %t (#{set_color(entry.friendly_size, :cyan)}) [#{set_color('%p%%', :green)}] [#{set_color('%B', :green)}] %e "
+
+        ProgressBar.create title:  entry.basename,
+                           total:  entry.size,
+                           smoothing:     0.5,
+                           throttle_rate: 0.3,
+                           format:     format,
+                           progress_mark: '·',
+                           starting_at: starting_at
+      end
+
+    end
+
+  end
+end