rsh2 0.0.1

data/Rakefile

@@ -0,0 +1,10 @@
+require 'rake_ext'
+
+project(
+  name: "rsh2",
+  version: "0.0.1",
+  summary: "ruby ssh and io",
+
+  author: "Alexey Petrushin",
+  homepage: "http://github.com/alexeypetrushin/rssh"
+)

data/lib/old/ssh.rb

@@ -0,0 +1,11 @@
+class Ssh
+  attr_reader :config
+  
+  
+  def setstat(path, options)
+    remote do |sh, sftp|
+      sftp.setstat! path, options
+    end
+  end
+  
+end

data/lib/rsh.rb

@@ -0,0 +1,18 @@
+raise 'ruby 1.9.2 or higher required!' if RUBY_VERSION < '1.9.2'
+
+require 'rsh/gems'
+
+require 'open3'
+
+require 'net/ssh'
+require 'net/sftp'
+
+%w(
+  support
+
+  drivers/abstract
+  drivers/local
+  drivers/ssh
+
+  box
+).each{|f| require "rsh/#{f}"}

data/lib/rsh/box.rb

@@ -0,0 +1,145 @@
+module Rsh
+  class Box
+    attr_accessor :options
+    
+    def initialize options = {}
+      @options = options
+      options[:host] ||= 'localhost'
+    end
+    
+    def driver
+      unless @driver
+        klass = options[:host] == 'localhost' ? Drivers::Local : Drivers::Ssh
+        @driver = klass.new options
+      end
+      @driver
+    end
+    
+    def local_driver
+      @local_driver ||= Drivers::Local.new
+    end
+    
+    def bulk &b
+      driver.bulk &b
+    end
+    
+    def upload_file from_local_path, to_remote_path, options = {}
+      bulk do
+        raise "file '#{from_local_path}' not exists!" unless local_driver.file_exist? from_local_path
+        if driver.file_exist?(to_remote_path)
+          if options[:override]
+            driver.remove_file to_remote_path
+          else
+            raise "file '#{to_remote_path}' already exists!"
+          end
+        end        
+        driver.upload_file from_local_path, to_remote_path
+      end
+    end
+    
+    def download_file from_remote_path, to_local_path, options = {}
+      bulk do
+        raise "file #{from_remote_path} not exists!" unless driver.file_exist?(from_remote_path)
+        if local_driver.file_exist? to_local_path
+          if options[:override]
+            local_driver.remove_file to_local_path
+          else
+            raise "file #{to_local_path} already exists!" 
+          end
+        end
+        driver.download_file from_remote_path, to_local_path
+      end
+    end
+    
+    def remove_file remote_file_path
+      bulk do
+        raise "file #{remote_file_path} not exists!" unless driver.file_exist? remote_file_path
+        driver.remove_file remote_file_path
+      end
+    end
+    
+    def exist? remote_path
+      driver.exist? remote_path
+    end
+    
+    def file_exist? remote_file_path
+      driver.file_exist? remote_file_path
+    end
+    
+    def remove_directory remote_directory_path
+      bulk do
+        raise "directory #{remote_directory_path} not exists!" unless driver.directory_exist? remote_directory_path
+        driver.remove_directory remote_directory_path
+      end
+    end
+    
+    def create_directory remote_path, options = {}
+      bulk do
+        if driver.directory_exist?(remote_path)
+          if options[:override]
+            driver.remove_directory remote_path
+          else
+            raise "directory '#{remote_path}' already exists!"
+          end
+        end
+        driver.create_directory remote_path
+      end
+    end
+    
+    def directory_exist? remote_path
+      driver.directory_exist? remote_path
+    end
+    
+    def upload_directory from_local_path, to_remote_path, options = {}
+      bulk do
+        raise "directory '#{from_local_path}' not exists!" unless local_driver.directory_exist? from_local_path
+        if driver.directory_exist?(to_remote_path)
+          if options[:override]
+            driver.remove_directory to_remote_path
+          else
+            raise "directory '#{to_remote_path}' already exists!"
+          end
+        end        
+        driver.upload_directory from_local_path, to_remote_path
+      end
+    end
+    
+    def download_directory from_remote_path, to_local_path, options = {}
+      bulk do
+        raise "directory #{from_remote_path} not exists!" unless driver.directory_exist?(from_remote_path)
+        if local_driver.directory_exist? to_local_path
+          if options[:override]
+            local_driver.remove_directory to_local_path
+          else
+            raise "directory #{to_local_path} already exists!" 
+          end
+        end
+        driver.download_directory from_remote_path, to_local_path
+      end
+    end
+    
+    def with_tmp_dir &block
+      bulk do
+        tmp_dir = driver.generate_tmp_dir_name
+        begin   
+          remove_directory tmp_dir if directory_exist? tmp_dir
+          create_directory tmp_dir
+          block.call
+        ensure
+          remove_directory tmp_dir if directory_exist? tmp_dir
+        end
+      end
+    end
+    
+    def bash cmd
+      code, stdout, stderr = driver.exec cmd
+      raise "can't execute '#{cmd}'!" unless code == 0
+      return stdout, stderr
+    end
+    
+    protected
+      def method_missing m, *a, &b
+        driver.send m, *a, &b
+      end
+  end
+end

data/lib/rsh/drivers/abstract.rb

@@ -0,0 +1,19 @@
+module Rsh
+  module Drivers
+    class Abstract
+      attr_reader :options
+    
+      def initialize options = {}
+        @options = options
+      end
+    
+      def bulk &b
+        b.call
+      end
+    
+      def generate_tmp_dir_name
+        "/tmp/ssh_tmp_dir_#{rand(10**6)}"
+      end
+    end
+  end
+end

data/lib/rsh/drivers/local.rb

@@ -0,0 +1,52 @@
+module Rsh
+  module Drivers
+    class Local < Abstract
+      def bulk &b
+        b.call
+      end
+    
+      def upload_file from_local_path, to_remote_path
+        FileUtils.copy from_local_path, to_remote_path
+      end
+    
+      def download_file from_remote_path, to_local_path
+        FileUtils.copy from_remote_path, to_local_path
+      end
+    
+      def exist? remote_file_path
+        File.exist? remote_file_path
+      end
+    
+      alias_method :directory_exist?, :exist?
+      alias_method :file_exist?, :exist?
+    
+      def remove_file remote_file_path
+        File.delete remote_file_path
+      end    
+    
+      def create_directory path
+        Dir.mkdir path
+      end
+    
+      def remove_directory path
+        FileUtils.rm_r path
+      end
+      
+      def upload_directory from_local_path, to_remote_path
+        FileUtils.cp_r from_local_path, to_remote_path
+      end
+      
+      def download_directory from_remote_path, to_local_path
+        FileUtils.cp_r from_remote_path, to_local_path
+      end
+    
+      def exec command        
+        code, stdout, stderr = Open3.popen3 command do |stdin, stdout, stderr, waitth|  
+          [waitth.value.to_i, stdout.read, stderr.read]
+        end
+      
+        return code, stdout, stderr
+      end
+    end
+  end
+end

data/lib/rsh/drivers/ssh.rb

@@ -0,0 +1,145 @@
+module Rsh
+  module Drivers
+    class Ssh < Abstract
+      def initialize options = {}
+        super
+        raise "ssh options not provided!" unless options[:ssh]
+        raise "invalid ssh options!" unless options[:ssh].is_a?(Hash)
+      end
+    
+      def upload_file from_local_path, to_remote_path
+        remote do |ssh, sftp|
+          sftp.upload! from_local_path, to_remote_path
+        end
+      end
+
+      def download_file from_remote_path, to_local_path
+        File.open to_local_path, "w" do |out|
+          remote do |ssh, sftp|
+            sftp.download! from_remote_path, out #, :recursive => true          
+          end
+        end
+      end
+
+      def exist? remote_file_path
+        remote do |ssh, sftp|
+          begin
+            fattrs = sftp.stat! remote_file_path
+            fattrs.directory? or fattrs.file? or fattrs.symlink?
+          rescue Net::SFTP::StatusException
+            false
+          end
+        end
+      end
+      alias_method :directory_exist?, :exist?
+      alias_method :file_exist?, :exist?
+
+      def remove_file remote_file_path
+        remote do |ssh, sftp|
+          sftp.remove! remote_file_path
+        end
+      end
+
+      def exec command
+        remote do |ssh, sftp|
+          # somehow net-ssh doesn't executes ~/.profile, so we need to execute it manually
+          # command = ". ~/.profile && #{command}"
+
+          stdout, stderr, code, signal = hacked_exec! ssh, command
+
+          return code, stdout, stderr
+        end
+      end    
+    
+      def open_connection      
+        ssh_options = self.options[:ssh].clone
+        host = options[:host] || raise('host not provided!')
+        user = ssh_options.delete(:user) || raise('user not provied!')
+        @ssh = Net::SSH.start(host, user, ssh_options)
+        @sftp = @ssh.sftp.connect
+      end
+    
+      def close_connection
+        @ssh.close
+        # @sftp.close not needed
+        @ssh, @sftp = nil
+      end
+    
+      def bulk &block
+        remote &block
+      end
+    
+      def create_directory path
+        remote do |ssh, sftp|          
+          sftp.mkdir! path
+          # exec "mkdir #{path}"
+        end        
+      end
+    
+      def remove_directory path
+        exec "rm -r #{path}"
+      end
+      
+      def upload_directory from_local_path, to_remote_path
+        remote do |ssh, sftp|
+          sftp.upload! from_local_path, to_remote_path
+        end
+      end
+      
+      def download_directory from_remote_path, to_local_path
+        remote do |ssh, sftp|
+          sftp.download! from_remote_path, to_local_path, :recursive => true
+        end
+      end
+    
+      protected
+        # taken from here http://stackoverflow.com/questions/3386233/how-to-get-exit-status-with-rubys-netssh-library/3386375#3386375
+        def hacked_exec!(ssh, command, &block)
+          stdout_data = ""
+          stderr_data = ""
+          exit_code = nil
+          exit_signal = nil
+        
+          channel = ssh.open_channel do |channel|
+            channel.exec(command) do |ch, success|
+              raise "could not execute command: #{command.inspect}" unless success
+
+              channel.on_data{|ch2, data| stdout_data << data}
+              channel.on_extended_data{|ch2, type, data| stderr_data << data}
+              channel.on_request("exit-status"){|ch,data| exit_code = data.read_long}
+              channel.on_request("exit-signal"){|ch, data| exit_signal = data.read_long}
+            end          
+          end  
+        
+          channel.wait      
+        
+          [stdout_data, stderr_data, exit_code, exit_signal]
+        end
+    
+        def remote(&block)
+          if @ssh
+            block.call @ssh, @sftp
+          else            
+            # Rails.logger.info "Connecting to remote Hadoop #{options[:user]}@#{options[:host]}"          
+            begin
+              open_connection
+              block.call @ssh, @sftp
+            ensure
+              close_connection
+            end
+                              
+            # Net::SSH.start options[:host], options[:user], :config => true do |ssh|
+            #   ssh.sftp.connect do |sftp|
+            #     begin
+            #       @ssh, @sftp = ssh, sftp
+            #       block.call @ssh, @sftp
+            #     ensure
+            #       @ssh, @sftp = nil
+            #     end
+            #   end
+            # end
+          end
+        end
+    end
+  end
+end

data/lib/rsh/gems.rb

@@ -0,0 +1,2 @@
+gem 'net-ssh'
+gem 'net-sftp'

data/lib/rsh/support.rb

@@ -0,0 +1,30 @@
+require 'fileutils'
+require 'net/ssh'
+require 'net/sftp'
+
+# class File
+#   class << self
+#     def ensure_dir_exist directory, options = {}       
+#       options = options.symbolize_keys 
+#       clean = options.delete :clean
+#       raise "unsupported options #{options.keys}!" unless options.empty?
+#     
+#       FileUtils.rm_r directory, :force => true if clean and File.exist?(directory)
+#       FileUtils.mkdir_p directory
+#     end
+#   
+#     def copy_dir_content from, to
+#       Dir.glob "#{from}/*" do |item|
+#         FileUtils.cp_r item, to
+#       end
+#     end
+#     
+#     def copy_dir from, to
+#       FileUtils.cp_r from, to
+#     end
+#     
+#     def remove_dir dir
+#       FileUtils.rm_r dir
+#     end
+#   end
+# end

data/readme.md

@@ -0,0 +1,4 @@
+TODO
+- introduce Entity/Dir/File (the same as in Rush)
+- allow to move files between any Boxes, not only between local and remote.
+- add support for moving dirs.

data/spec/abstract_driver.rb

@@ -0,0 +1,82 @@
+require 'spec_helper'
+
+shared_examples_for 'abstract driver' do
+  dir = "#{File.dirname __FILE__}/abstract_driver"
+  with_tmp_spec_dir dir, before: :each
+  
+  describe "io" do
+    before :each do
+      @local_dir = spec_dir      
+      @remote_dir = @driver.generate_tmp_dir_name    
+      
+      @driver.remove_directory @remote_dir if @driver.directory_exist? @remote_dir
+      @driver.create_directory @remote_dir
+    end
+
+    after :each do
+      @driver.remove_directory @remote_dir if @driver.directory_exist? @remote_dir
+    end
+  
+    describe "files" do  
+      before :each do
+        @local_file = "#{@local_dir}/local_file"
+        @check_file = "#{@local_dir}/check_file"
+        @remote_file = "#{@remote_dir}/remote_file"
+      end
+            
+      it "file_exist?" do
+        @driver.file_exist?(@remote_file).should be_false
+        @driver.upload_file(@local_file, @remote_file)
+        @driver.file_exist?(@remote_file).should be_true
+      end
+
+      it "upload & download file" do
+        @driver.upload_file(@local_file, @remote_file)
+        @driver.file_exist?(@remote_file).should be_true
+      
+        @driver.download_file(@remote_file, @check_file)
+        File.read(@local_file).should == File.read(@check_file)
+      end
+    
+      it "remove_file" do
+        @driver.upload_file(@local_file, @remote_file)
+        @driver.file_exist?(@remote_file).should be_true
+        @driver.remove_file(@remote_file)
+        @driver.file_exist?(@remote_file).should be_false
+      end
+    end
+    
+    describe 'directories' do
+      before :each do
+        @from_local, @remote_path, @to_local = "#{@local_dir}/dir", "#{@remote_dir}/upload", "#{@local_dir}/download"
+      end
+      
+      it "directory_exist?, create_directory, remove_directory" do
+        dir = "#{@remote_dir}/some_dir"
+        @driver.directory_exist?(dir).should be_false
+        @driver.create_directory(dir)
+        @driver.directory_exist?(dir).should be_true
+        @driver.remove_directory(dir)
+        @driver.directory_exist?(dir).should be_false
+      end
+    
+      it "upload_directory & download_directory" do
+        upload_path_check = "#{@remote_path}/dir2/file"
+        @driver.file_exist?(upload_path_check).should be_false
+        @driver.upload_directory(@from_local, @remote_path)
+        @driver.file_exist?(upload_path_check).should be_true
+
+        download_path_check = "#{@to_local}/dir2/file"
+        File.exist?(download_path_check).should be_false
+        @driver.download_directory(@remote_path, @to_local)
+        File.exist?(download_path_check).should be_true
+      end
+    end  
+  end
+  
+  describe "shell" do
+    it 'exec' do
+      @driver.exec("echo 'ok'").should == [0, "ok\n", ""]
+    end  
+  end
+end

data/spec/abstract_driver/local_file

@@ -0,0 +1 @@
+some content

data/spec/box_spec.rb

@@ -0,0 +1,105 @@
+require 'spec_helper'
+
+describe Rsh::Box do  
+  with_tmp_spec_dir before: :each
+  
+  before :each do
+    @box = Rsh::Box.new
+
+    @local_dir = spec_dir    
+    @remote_dir = @box.generate_tmp_dir_name        
+    
+    @box.remove_directory @remote_dir if @box.directory_exist? @remote_dir
+    @box.create_directory @remote_dir
+  end
+
+  after :each do
+    @box.remove_directory @remote_dir if @box.directory_exist? @remote_dir
+  end
+  
+  describe "io" do
+    describe "files" do
+      before :each do
+        @local_file = "#{@local_dir}/local_file"
+        @check_file = "#{@local_dir}/check_file"
+        @remote_file = "#{@remote_dir}/remote_file"
+      end
+      
+      it "file_exist?" do
+        @box.file_exist?(@remote_file).should be_false
+        @box.upload_file(@local_file, @remote_file)
+        @box.file_exist?(@remote_file).should be_true
+      end
+
+      it "upload_file" do
+        @box.upload_file(@local_file, @remote_file)
+        @box.file_exist?(@remote_file).should be_true
+
+        lambda{@box.upload_file(@local_file, @remote_file)}.should raise_error(/exists/)
+
+        # upload with override
+        @box.upload_file(@local_file, @remote_file, override: true)
+        @box.file_exist?(@remote_file).should be_true
+      end
+
+      it "download_file" do
+        lambda{@box.download_file(@remote_file, @check_file)}.should raise_error(/not exists/)
+        @box.upload_file(@local_file, @remote_file)
+        @box.download_file(@remote_file, @check_file)
+        File.read(@local_file).should == File.read(@check_file)
+      end          
+
+      it "remove_file" do
+        lambda{@box.remove_file(@remote_file)}.should raise_error(/not exists/)
+        @box.upload_file(@local_file, @remote_file)
+        @box.file_exist?(@remote_file).should be_true
+        @box.remove_file(@remote_file)
+        @box.file_exist?(@remote_file).should be_false
+      end
+    end
+    
+    describe 'directories' do
+      before :each do 
+        @from_local, @remote_path, @to_local = "#{@local_dir}/dir", "#{@remote_dir}/upload", "#{@local_dir}/download"
+      end
+      
+      it "directory_exist?" do
+        @box.file_exist?(@remote_path).should be_false
+        @box.upload_directory(@from_local, @remote_path)
+        @box.file_exist?(@remote_path).should be_true
+      end
+
+      it "upload_directory" do
+        @box.upload_directory(@from_local, @remote_path)
+        @box.directory_exist?(@remote_path).should be_true
+    
+        lambda{@box.upload_directory(@from_local, @remote_path)}.should raise_error(/exists/)
+    
+        # upload with override
+        @box.upload_directory(@from_local, @remote_path, override: true)
+        @box.directory_exist?(@remote_path).should be_true
+      end
+    
+      it "download_directory" do
+        lambda{@box.download_directory(@remote_path, @to_local)}.should raise_error(/not exists/)
+        @box.upload_directory(@from_local, @remote_path)
+        @box.download_directory(@remote_path, @to_local)
+        File.exist?("#{@to_local}/dir2/file").should be_true
+      end          
+    
+      it "remove_directory" do
+        lambda{@box.remove_directory(@remote_path)}.should raise_error(/not exists/)
+        @box.upload_directory(@from_local, @remote_path)
+        @box.directory_exist?(@remote_path).should be_true
+        @box.remove_directory(@remote_path)
+        @box.directory_exist?(@remote_path).should be_false
+      end    
+    end
+  end
+  
+  describe "shell" do
+    it 'exec' do
+      @box.bash("echo 'ok'").should == ["ok\n", ""]
+    end  
+  end
+end

data/spec/box_spec/local_file

@@ -0,0 +1 @@
+some content

data/spec/config.example.yml

@@ -0,0 +1,4 @@
+:ssh:
+  :host: some_host.com
+  :user: some_user
+  :password: some_password

data/spec/config.yml

@@ -0,0 +1,5 @@
+:remote_driver:
+  :host: universal.4ire.net
+  :ssh:    
+    :user: root
+    :password: privateprivate

data/spec/local_driver_spec.rb

@@ -0,0 +1,9 @@
+require 'abstract_driver'
+
+describe Rsh::Drivers::Local do
+  it_should_behave_like "abstract driver"    
+    
+  before :each do
+    @driver = Rsh::Drivers::Local.new
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,11 @@
+require 'rspec_ext'
+require 'rsh'
+
+rspec do
+  def config
+    dir = File.dirname __FILE__
+    config_file_path = "#{dir}/config.yml"
+    raise "no config '#{config_file_path}'!" unless File.exist? config_file_path
+    @config ||= YAML.load_file config_file_path
+  end
+end

data/spec/ssh_driver_spec.rb

@@ -0,0 +1,15 @@
+require 'abstract_driver'
+require 'yaml'
+
+describe Rsh::Drivers::Ssh do
+  it_should_behave_like "abstract driver"    
+
+  before :each do
+    @driver = Rsh::Drivers::Ssh.new config[:remote_driver]
+    @driver.open_connection
+  end
+  
+  after :each do
+    @driver.close_connection
+  end  
+end

metadata

@@ -0,0 +1,84 @@
+--- !ruby/object:Gem::Specification 
+name: rsh2
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Alexey Petrushin
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-01-30 00:00:00 +03:00
+default_executable: 
+dependencies: []
+
+description: 
+email: 
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- Rakefile
+- readme.md
+- lib/old/ssh.rb
+- lib/rsh/box.rb
+- lib/rsh/drivers/abstract.rb
+- lib/rsh/drivers/local.rb
+- lib/rsh/drivers/ssh.rb
+- lib/rsh/gems.rb
+- lib/rsh/support.rb
+- lib/rsh.rb
+- spec/abstract_driver/dir/dir2/file
+- spec/abstract_driver/local_file
+- spec/abstract_driver.rb
+- spec/box_spec/dir/dir2/file
+- spec/box_spec/local_file
+- spec/box_spec.rb
+- spec/config.example.yml
+- spec/config.yml
+- spec/local_driver_spec.rb
+- spec/spec_helper.rb
+- spec/ssh_driver_spec.rb
+has_rdoc: true
+homepage: http://github.com/alexeypetrushin/rssh
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: ruby ssh and io
+test_files: []
+