drbqs 0.0.15 → 0.0.16

data/lib/drbqs/task/task_generator.rb

@@ -1,142 +1,119 @@
 module DRbQS
-  class DRbQS::TaskCreatingError < StandardError
-  end
-
-  class TaskSource
-    def initialize(data)
-      data.each do |key, val|
-        instance_variable_set("@#{key.to_s}", val)
+  class Task
+    class Generator
+      # @param [Hash] data Names of instance variables and their values,
+      #  which can be accessed in {DRbQS::Task::Generator#set}.
+      def initialize(data = {})
+        @registrar = DRbQS::Task::Registrar.new(data)
+        @fiber = nil
+        @iterate = nil
+        @task_set = nil
+        @fiber_init = nil
+        @wait = false
       end
-    end
 
-    def add_task(arg)
-      case arg
-      when DRbQS::Task
-        Fiber.yield(arg)
-      when Array
-        arg.each { |t| Fiber.yield(t) }
-      else
-        raise "Invalid type of an argument."
+      def have_next?
+        !!@fiber
       end
-    end
 
-    def create_add_task(*args, &block)
-      add_task(DRbQS::Task.new(*args, &block))
-    end
-
-    def wait_all_tasks
-      Fiber.yield(:wait)
-    end
-  end
-
-  class TaskGenerator
-    def initialize(data = {})
-      @source = DRbQS::TaskSource.new(data)
-      @fiber = nil
-      @iterate = nil
-      @task_set = nil
-      @fiber_init = nil
-      @wait = false
-    end
-
-    def have_next?
-      !!@fiber
-    end
-
-    def waiting?
-      @wait
-    end
-
-    # The options :generate and :collect are available.
-    # opts[:generate] is the number of tasks per one generation.
-    # The generator creates a task set from opts[:collect] tasks.
-    def set(opts = {}, &block)
-      @iterate = opts[:generate] || 1
-      @task_set = opts[:collect]
-      if @iterate < 1 || (@task_set && @task_set < 1)
-        raise ArgumentError, "Invalid options of task creation on generator."
+      def waiting?
+        @wait
       end
-      @fiber_init = lambda do
-        @fiber = Fiber.new do
-          begin
-            @source.instance_eval(&block)
-          rescue => err
-            new_err = DRbQS::TaskCreatingError.new("#{err.to_s} (#{err.class}) when creating task")
-            new_err.set_backtrace(err.backtrace)
-            raise new_err
+
+      # @param [Hash] opts The options of task generation
+      # @option opts [Fixnum] :generate Set the number of tasks per one generation
+      # @option opts [Fixnum] :collect The generator creates a task set consisting of opts[:collect] tasks.
+      def set(opts = {}, &block)
+        unless block_given?
+          raise ArgumentError, "Creation of a task generator needs block."
+        end
+        @iterate = opts[:generate] || 1
+        @task_set = opts[:collect]
+        if @iterate < 1 || (@task_set && @task_set < 1)
+          raise ArgumentError, "Invalid options of task creation on generator."
+        end
+        @fiber_init = lambda do
+          @fiber = Fiber.new do
+            begin
+              @registrar.instance_eval(&block)
+            rescue => err
+              new_err = self.class.new("Error on generating tasks: #{err.to_s} (#{err.class})")
+              new_err.set_backtrace(err.backtrace)
+              raise new_err
+            end
+            nil
           end
-          nil
         end
       end
-    end
 
-    # Initialize fider to create tasks.
-    # This method must be called in thread to create tasks.
-    def init
-      @fiber_init.call if @fiber_init
-    end
+      # Initialize fider to create tasks.
+      # This method must be called in thread to create tasks.
+      def init
+        @fiber_init.call if @fiber_init
+      end
 
-    # Return an array of new tasks.
-    def new_tasks
-      if @fiber
-        @wait = false
-        task_ary = []
-        iteration = @iterate
-        iteration *= @task_set if @task_set
-        iteration.times do |i|
-          if task_new = @fiber.resume
-            case task_new
-            when DRbQS::Task
-              task_ary << task_new
-            when Array
-              task_ary.concat(task_new)
-            when :wait
-              @wait = true
-              break
+      # Return an array of new tasks.
+      def new_tasks
+        if @fiber
+          @wait = false
+          task_ary = []
+          iteration = @iterate
+          iteration *= @task_set if @task_set
+          iteration.times do |i|
+            if task_new = @fiber.resume
+              case task_new
+              when DRbQS::Task
+                task_ary << task_new
+              when Array
+                task_ary.concat(task_new)
+              when :wait
+                @wait = true
+                break
+              else
+                raise RuntimeError, "Invalid object created by fiber to create tasks."
+              end
             else
-              raise "Invalid type of new task."
+              @fiber = nil
+              break
             end
-          else
-            @fiber = nil
-            break
           end
-        end
-        if task_ary.size > 0
-          if @task_set
-            task_ary = task_ary.each_slice(@task_set).map do |ary|
-              DRbQS::TaskSet.new(ary)
+          if task_ary.size > 0
+            if @task_set
+              task_ary = task_ary.each_slice(@task_set).map do |ary|
+                DRbQS::Task::TaskSet.new(ary)
+              end
             end
+            return task_ary
           end
-          return task_ary
         end
+        nil
       end
-      nil
-    end
 
-    DEBUG_TASK_PROGRESS = 1000
+      DEBUG_TASK_PROGRESS = 1000
 
-    # Create all tasks for test and return [group_number, task_number] if all tasks created properly.
-    def debug_all_tasks(opts = {})
-      limit = opts[:limit]
-      progress = opts[:progress]
-      group_number = 0
-      task_number = 0
-      while ary = new_tasks
-        ary.each do |t|
-          unless DRbQS::Task === t
-            raise "Invalid #{i}th task: #{t.inspect}"
-          end
-          task_number += 1
-          if progress && (task_number % DEBUG_TASK_PROGRESS == 0)
-            puts "#{task_number} tasks have been created."
-          end
-          if limit && task_number > limit
-            break
+      # Create all tasks for test and return [group_number, task_number] if all tasks created properly.
+      def debug_all_tasks(opts = {})
+        limit = opts[:limit]
+        progress = opts[:progress]
+        group_number = 0
+        task_number = 0
+        while ary = new_tasks
+          ary.each do |t|
+            unless DRbQS::Task === t
+              raise RuntimeError, "Invalid #{i}th task: #{t.inspect}"
+            end
+            task_number += 1
+            if progress && (task_number % DEBUG_TASK_PROGRESS == 0)
+              puts "#{task_number} tasks have been created."
+            end
+            if limit && task_number > limit
+              break
+            end
           end
+          group_number += 1
         end
-        group_number += 1
+        [group_number, task_number]
       end
-      [group_number, task_number]
     end
   end
 end

data/lib/drbqs/utility/misc.rb

@@ -1,21 +1,21 @@
 require 'sys/proctable'
 
 module DRbQS
-  class LoggerDummy
-    def info(*args)
-    end
+  module Misc
+    class LoggerDummy
+      def info(*args)
+      end
 
-    def warn(*args)
-    end
+      def warn(*args)
+      end
 
-    def error(*args)
-    end
+      def error(*args)
+      end
 
-    def debug(*args)
+      def debug(*args)
+      end
     end
-  end
 
-  module Misc
     # :port
     # :host
     # :unix
@@ -56,6 +56,11 @@ module DRbQS
     end
     module_function :time_to_history_string
 
+    def time_to_history_string2(t)
+      t.strftime("%m-%d %H:%M:%S")
+    end
+    module_function :time_to_history_string2
+
     STRINGS_FOR_KEY = ('a'..'z').to_a + ('A'..'Z').to_a + ('0'..'9').to_a
 
     def random_key(size = 20)

data/lib/drbqs/utility/temporary.rb

@@ -22,8 +22,13 @@ module DRbQS
     end
 
     # Return new path of temporary file.
-    def self.file
-      filename.create(:add => :always)
+    # @param [String] basename Set the basename of created filename
+    def self.file(basename = nil)
+      if basename
+        File.join(self.directory, basename)
+      else
+        filename.create(:add => :always)
+      end
     end
 
     # Make root of temporary directory empty.

data/lib/drbqs/utility/transfer/transfer.rb

@@ -0,0 +1,81 @@
+module DRbQS
+  # To compress files, we use gzip and tar command.
+  # Note that if we compress files then we delete the original files.
+  class Transfer
+    @files = Queue.new
+
+    class << self
+      # Add path to queue of which files is going to be transfered to server.
+      # @param [String] path The file path that we want to send to a server.
+      # @param [Hash] opts The options for transfering a file.
+      # @option opts [true,false] :compress Compress the file by gzip before transfering.
+      # @option opts [String] :rename Change basename to the specified name.
+      def enqueue(path, opts = {})
+        if opts[:rename]
+          new_path = FileName.create(File.join(File.dirname(path), opts[:rename]), :directory => :parent)
+          FileUtils.mv(path, new_path)
+          path = new_path
+        end
+        if opts[:compress]
+          if File.directory?(path)
+            gz_path = "#{path.sub(/\/$/, '')}.tar.gz"
+            cmd = "tar czf #{gz_path} -C #{File.dirname(path)} #{File.basename(path)} > /dev/null 2>&1"
+          else
+            gz_path = path + '.gz'
+            cmd = "gzip --best #{path} > /dev/null 2>&1"
+          end
+          if File.exist?(gz_path)
+            raise "File has already existed: #{gz_path}"
+          elsif !system(cmd)
+            raise "Can not compress: #{path}"
+          end
+          FileUtils.rm_r(path) if File.exist?(path)
+          path_to_send = gz_path
+        else
+          path_to_send = path
+        end
+        @files.enq(path_to_send)
+        File.basename(path_to_send)
+      end
+
+      def compress_enqueue(path)
+        enqueue(path, :compress => true)
+      end
+
+      def dequeue
+        @files.deq
+      end
+
+      def empty?
+        @files.empty?
+      end
+
+      def dequeue_all
+        files = []
+        until empty?
+          files << dequeue
+        end
+        files.empty? ? nil : files
+      end
+
+      # Decompress a file in the file directory of a server.
+      # @param [DRbQS::Server] server Current server
+      # @param [String] filename File path to decompress
+      def decompress(server, filename)
+        dir = server.transfer_directory
+        path = File.join(dir, filename)
+        if File.exist?(path)
+          case path
+          when /\.tar\.gz$/
+            cmd = "tar xvzf #{path} -C #{dir} > /dev/null 2>&1"
+          when /\.gz$/
+            cmd = "gunzip #{path} > /dev/null 2>&1"
+          else
+            cmd = nil
+          end
+          system(cmd) if cmd
+        end
+      end
+    end
+  end
+end

data/lib/drbqs/utility/transfer/transfer_client.rb

@@ -1,90 +1,89 @@
-require 'net/sftp'
+require 'drbqs/utility/transfer/transfer_client_connect.rb'
 
 module DRbQS
+  class Transfer
+    class Client
+      attr_reader :directory, :local, :sftp
 
-  class TransferClient
-
-    class ClientBase
-      def initialize(directory)
-        @directory = File.expand_path(directory)
-      end
-
-      def upload_name(path)
-        File.join(@directory, File.basename(path))
-      end
-      private :upload_name
-    end
-
-    # Transfer files to directory on DRbQS server over sftp.
-    # Note that after we transfer files we delete the original files.
-    class SFTP < DRbQS::TransferClient::ClientBase
-      attr_reader :user, :host, :directory
-
-      def initialize(user, host, directory)
-        super(directory)
-        @user = user
-        @host = host
+      def initialize(server_directory, same_host, host, user)
+        unless Pathname.new(server_directory).absolute?
+          raise ArgumentError, "Directory of server must be absolute."
+        end
+        @directory = server_directory
+        @same_host = same_host
+        @local = DRbQS::Transfer::Client::Local.new(@directory)
+        if host && user
+          @sftp = DRbQS::Transfer::Client::SFTP.new(user, host, @directory)
+        else
+          @sftp = nil
+        end
       end
 
-      # Transfer and delete +files+.
       def transfer(files)
-        Net::SFTP.start(@host, @user) do |sftp|
-          files.each do |path|
-            sftp.upload(path, upload_name(path))
-            FileUtils.rm_r(path)
+        transfered = false
+        if @same_host
+          begin
+            @local.transfer(files)
+            transfered = true
+          rescue
+          end
+        end
+        if !transfered
+          unless @sftp
+            raise "Can not transfer files."
           end
+          @sftp.transfer(files)
         end
-      rescue => err
-        raise err.class, "user=#{@user}, host=#{@host}, directory=#{@directory}; #{err.to_s}", err.backtrace 
       end
-    end
 
-    class Local < DRbQS::TransferClient::ClientBase
-      def transfer(files)
-        files.each do |path|
-          FileUtils.mv(path, upload_name(path))
+      def download(files, readonly = nil)
+        download_files = nil
+        if @same_host
+          begin
+            if readonly
+              download_files = files
+            else
+              download_files = @local.download(files)
+            end
+          rescue
+          end
         end
+        if !download_files
+          unless @sftp
+            raise "SFTP is not prepared."
+          end
+          download_files = @sftp.download(files)
+        end
+        download_files
       end
-    end
-
-    attr_reader :directory, :local, :sftp
-
-    def initialize(dir)
-      @directory = File.expand_path(dir)
-      @local = DRbQS::TransferClient::Local.new(@directory)
-      @sftp = nil
-    end
-
-    def make_directory
-      FileUtils.mkdir_p(@directory)
-    end
 
-    def set_sftp(user, host)
-      @sftp = DRbQS::TransferClient::SFTP.new(user, host, @directory)
-    end
+      class << self
+        @transfer = nil
 
-    def information
-      info = "directory: #{@directory}"
-      info << ", sftp: #{@sftp.user}@#{@sftp.host}" if @sftp
-      info
-    end
+        def get
+          @transfer
+        end
 
-    def transfer(files, on_same_host = nil)
-      transfered = false
-      if on_same_host
-        begin
-          @local.transfer(files)
-          transfered = true
-        rescue
+        def set(transfer)
+          @transfer = transfer
         end
-      end
-      if !transfered
-        unless @sftp
-          raise "Can not transfer files."
+
+        def transfer_to_server
+          if files = DRbQS::Transfer.dequeue_all
+            if @transfer
+              begin
+                @transfer.transfer(files)
+              rescue Exception => err
+                err_new = err.class.new("#{err.to_s} (#{err.class}); Can not send file: #{files.join(", ")}")
+                err_new.set_backtrace(err.backtrace)
+                raise err_new
+              end
+            else
+              raise "Server does not set transfer settings. Can not send file: #{files.join(", ")}"
+            end
+          end
         end
-        @sftp.transfer(files)
       end
     end
   end
-
 end