ztk 0.0.3 → 0.0.4

data/lib/ztk/logger.rb

@@ -41,7 +41,7 @@ module ZTK
         file = Regexp.last_match[1]
         line = Regexp.last_match[2]
         method = Regexp.last_match[3]
-        "#{File.basename(file)}:#{line}:#{method} | "
+        "#{File.basename(file)}:#{line}:#{method}|"
       else
         ""
       end
@@ -54,9 +54,9 @@ module ZTK
 
       called_by = parse_caller(caller[1])
       msg = (block && block.call)
-      return if (msg.nil? || msg.strip.empty?)
-      message = [message, progname, msg].delete_if{|i| i == nil}.join(": ")
-      message = "%19s.%06d+%05d|%5s|%s%s\n" % [Time.now.utc.strftime("%Y-%m-%d %H:%M:%S"), Time.now.utc.usec, Process.pid, SEVERITIES[severity], called_by, message]
+      (msg.nil? || msg.strip.empty?) and return
+      message = [message, progname, msg].delete_if{ |i| i.nil? }.join(": ")
+      message = "%19s.%06d|%05d|%5s|%s%s\n" % [Time.now.utc.strftime("%Y-%m-%d|%H:%M:%S"), Time.now.utc.usec, Process.pid, SEVERITIES[severity], called_by, message]
 
       @logdev.write(message)
 

data/lib/ztk/parallel.rb

@@ -2,9 +2,9 @@
 #
 #      Author: Zachary Patten <zachary@jovelabs.com>
 #   Copyright: Copyright (c) Jove Labs
-#     License: Apache License, Version 2.0
+#     License: Apache License, Vers::IOn 2.0
 #
-#   Licensed under the Apache License, Version 2.0 (the "License");
+#   Licensed under the Apache License, Vers::IOn 2.0 (the "License");
 #   you may not use this file except in compliance with the License.
 #   You may obtain a copy of the License at
 #
@@ -12,61 +12,81 @@
 #
 #   Unless required by applicable law or agreed to in writing, software
 #   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
+#   WITHOUT WARRANTIES OR CONDIT::IONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permiss::IOns and
+#   limitat::IOns under the License.
 #
 ################################################################################
+require "base64"
+require "ostruct"
 
 module ZTK
   class Parallel
 
 ################################################################################
 
-    attr_accessor :results
+    attr_accessor :config, :results
 
 ################################################################################
 
-    def initialize(options={})
-      GC.copy_on_write_friendly = true if GC.respond_to?(:copy_on_write_friendly=)
-
-      options.reverse_merge!(
-        :max_forks => `grep -c processor /proc/cpuinfo`.strip.to_i,
-        :one_shot => false
-      )
+    def initialize(config={})
+      @config = OpenStruct.new({
+        :stdout => $stdout,
+        :stderr => $stderr,
+        :stdin => $stdin,
+        :logger => $logger,
+        :max_forks => `grep -c processor /proc/cpuinfo`.chomp.to_i,
+        :one_shot => false,
+        :before_fork => nil,
+        :after_fork => nil
+      }.merge(config))
+      @config.stdout.sync = true if @config.stdout.respond_to?(:sync=)
+      @config.stderr.sync = true if @config.stderr.respond_to?(:sync=)
+      @config.stdin.sync = true if @config.stdin.respond_to?(:sync=)
+      @config.logger.sync = true if @config.logger.respond_to?(:sync=)
+
+      @forks = ::Array.new
+      @results = ::Array.new
+      ::GC.respond_to?(:copy_on_write_friendly=) and ::GC.copy_on_write_friendly = true
+    end
 
-      @max_forks = options[:max_forks]
-      @one_shot = options[:one_shot]
+################################################################################
 
-      @forks = Array.new
-      @results = Array.new
+    def config(&block)
+      if block_given?
+        yield(@config)
+      else
+        @config
+      end
     end
 
 ################################################################################
 
-    def process
+    def process(*args)
+      @config.logger.debug{ "FORKS #{@forks.inspect}" }
       pid = nil
-      return pid if (@forks.count >= @max_forks)
+      return pid if (@forks.count >= @config.max_forks)
 
-      child_reader, parent_writer = IO.pipe
-      parent_reader, child_writer = IO.pipe
+      child_reader, parent_writer = ::IO.pipe
+      parent_reader, child_writer = ::IO.pipe
 
-      ActiveRecord::Base.connection.disconnect!
-      pid = Process.fork do
-        ActiveRecord::Base.establish_connection
+      @config.before_fork and @config.before_fork.call(::Process.pid)
+      pid = ::Process.fork do
+        @config.after_fork and @config.after_fork.call(::Process.pid)
 
         parent_writer.close
         parent_reader.close
 
-        if (data = yield).present?
-          child_writer.write(Base64.encode64(Marshal.dump(data)))
+        if !(data = yield).nil?
+          @config.logger.debug{ "WRITE #{data.inspect}" }
+          child_writer.write(::Base64.encode64(::Marshal.dump(data)))
         end
 
         child_reader.close
         child_writer.close
-        Process.exit!(0)
+        ::Process.exit!(0)
       end
-      ActiveRecord::Base.establish_connection
+      @config.after_fork and @config.after_fork.call(::Process.pid)
 
       child_reader.close
       child_writer.close
@@ -80,18 +100,17 @@ module ZTK
 ################################################################################
 
     def wait
-      pid, status = (Process.wait2(-1, Process::WNOHANG) rescue nil)
-      if pid.present? && status.present?
-        if (fork = @forks.select{ |f| f[:pid] == pid }.first).present?
-          data = (Marshal.load(Base64.decode64(fork[:reader].read.to_s)) rescue nil)
-          @results.push(data) if (data.present? && !@one_shot)
-
-          fork[:reader].close
-          fork[:writer].close
-
-          @forks -= [fork]
-          return [pid, status, data]
-        end
+      @config.logger.debug{ "FORKS #{@forks.inspect}" }
+      pid, status = (::Process.wait2(-1, ::Process::WNOHANG) rescue nil)
+      if !pid.nil? && !status.nil? && !(fork = @forks.select{ |f| f[:pid] == pid }.first).nil?
+        data = (::Marshal.load(::Base64.decode64(fork[:reader].read.to_s)) rescue nil)
+        @config.logger.debug{ "READ #{data.inspect}" }
+        @results.push(data) if (!data.nil? && !@config.one_shot)
+        fork[:reader].close
+        fork[:writer].close
+
+        @forks -= [fork]
+        return [pid, status, data]
       end
       nil
     end
@@ -99,11 +118,11 @@ module ZTK
 ################################################################################
 
     def waitall
-      results = Array.new
+      data = ::Array.new
       while @forks.count > 0
-        results << wait
+        data << self.wait
       end
-      results
+      data
     end
 
 ################################################################################

data/lib/ztk/ssh.rb

@@ -17,6 +17,7 @@
 #   limitations under the License.
 #
 ################################################################################
+require "ostruct"
 
 module ZTK
   class SSHError < Error; end
@@ -28,21 +29,24 @@ module ZTK
 
 ################################################################################
 
-    def initialize(logger=$logger, stdout=$stdout, stderr=$stderr, stdin=$stdin)
-      @logger, @stdout, @stderr, @stdin = logger, stdout, stderr, stdin
-
-      @logger.sync = true if @logger.respond_to?(:sync=)
-      @stdout.sync = true if @stdout.respond_to?(:sync=)
-      @stderr.sync = true if @stderr.respond_to?(:sync=)
-      @stdin.sync = true if @stdin.respond_to?(:sync=)
-
-      @config = Hash.new(nil)
+    def initialize(config={})
+      @config = OpenStruct.new({
+        :stdout => $stdout,
+        :stderr => $stderr,
+        :stdin => $stdin,
+        :logger => $logger,
+        :ssh => Hash.new(nil)
+      }.merge(config))
+      @config.stdout.sync = true if @config.stdout.respond_to?(:sync=)
+      @config.stderr.sync = true if @config.stderr.respond_to?(:sync=)
+      @config.stdin.sync = true if @config.stdin.respond_to?(:sync=)
+      @config.logger.sync = true if @config.logger.respond_to?(:sync=)
     end
 
 ################################################################################
 
     def console
-      @logger and @logger.debug { "config(#{@config.inspect})" }
+      @config.logger and @config.logger.debug { "config(#{@config.ssh.inspect})" }
 
       command = [ "ssh" ]
       command << [ "-q" ]
@@ -50,47 +54,47 @@ module ZTK
       command << [ "-o", "StrictHostKeyChecking=no" ]
       command << [ "-o", "KeepAlive=yes" ]
       command << [ "-o", "ServerAliveInterval=60" ]
-      command << [ "-i", @config[:identity_file] ] if @config[:identity_file]
-      command << [ "-o", "ProxyCommand=\"#{proxy_command}\"" ] if @config[:proxy]
-      command << "#{@config[:ssh_user]}@#{@config[:host]}"
+      command << [ "-i", @config.ssh[:identity_file] ] if @config.ssh[:identity_file]
+      command << [ "-o", "ProxyCommand=\"#{proxy_command}\"" ] if @config.ssh[:proxy]
+      command << "#{@config.ssh[:ssh_user]}@#{@config.ssh[:host]}"
       command = command.flatten.compact.join(" ")
-      @logger and @logger.info { "command(#{command})" }
+      @config.logger and @config.logger.info { "command(#{command})" }
       Kernel.exec(command)
     end
 
 ################################################################################
 
     def exec(command, options={})
-      @ssh ||= Net::SSH.start(@config[:host], @config[:ssh_user], ssh_options)
+      @ssh ||= Net::SSH.start(@config.ssh[:host], @config.ssh[:ssh_user], ssh_options)
 
       options = { :silence => false }.merge(options)
       silence = options[:silence]
       output = ""
 
-      @logger and @logger.debug { "config(#{@config.inspect})" }
-      @logger and @logger.debug { "options(#{options.inspect})" }
-      @logger and @logger.info { "command(#{command})" }
+      @config.logger and @config.logger.debug { "config(#{@config.ssh.inspect})" }
+      @config.logger and @config.logger.debug { "options(#{options.inspect})" }
+      @config.logger and @config.logger.info { "command(#{command})" }
       channel = @ssh.open_channel do |chan|
-        @logger and @logger.debug { "channel opened" }
+        @config.logger and @config.logger.debug { "channel opened" }
         chan.exec(command) do |ch, success|
           raise SSHError, "Could not execute '#{command}'." unless success
 
           ch.on_data do |c, data|
             output += data
-            @logger and @logger.debug { data.chomp.strip }
-            @stdout.print(data) if !silence
+            @config.logger and @config.logger.debug { data.chomp.strip }
+            @config.stdout.print(data) if !silence
           end
 
           ch.on_extended_data do |c, type, data|
             output += data
-            @logger and @logger.debug { data.chomp.strip }
-            @stderr.print(data) if !silence
+            @config.logger and @config.logger.debug { data.chomp.strip }
+            @config.stderr.print(data) if !silence
           end
 
         end
       end
       channel.wait
-      @logger and @logger.debug { "channel closed" }
+      @config.logger and @config.logger.debug { "channel closed" }
 
       output
     end
@@ -98,22 +102,22 @@ module ZTK
 ################################################################################
 
     def upload(local, remote)
-      @sftp ||= Net::SFTP.start(@config[:host], @config[:ssh_user], ssh_options)
+      @sftp ||= Net::SFTP.start(@config.ssh[:host], @config.ssh[:ssh_user], ssh_options)
 
-      @logger and @logger.debug { "config(#{@config.inspect})" }
-      @logger and @logger.info { "parameters(#{local},#{remote})" }
+      @config.logger and @config.logger.debug { "config(#{@config.ssh.inspect})" }
+      @config.logger and @config.logger.info { "parameters(#{local},#{remote})" }
       @sftp.upload!(local.to_s, remote.to_s) do |event, uploader, *args|
         case event
         when :open
-          @logger and @logger.info { "upload(#{args[0].local} -> #{args[0].remote})" }
+          @config.logger and @config.logger.info { "upload(#{args[0].local} -> #{args[0].remote})" }
         when :close
-          @logger and @logger.debug { "close(#{args[0].remote})" }
+          @config.logger and @config.logger.debug { "close(#{args[0].remote})" }
         when :mkdir
-          @logger and @logger.debug { "mkdir(#{args[0]})" }
+          @config.logger and @config.logger.debug { "mkdir(#{args[0]})" }
         when :put
-          @logger and @logger.debug { "put(#{args[0].remote}, size #{args[2].size} bytes, offset #{args[1]})" }
+          @config.logger and @config.logger.debug { "put(#{args[0].remote}, size #{args[2].size} bytes, offset #{args[1]})" }
         when :finish
-          @logger and @logger.info { "finish" }
+          @config.logger and @config.logger.info { "finish" }
         end
       end
     end
@@ -121,22 +125,22 @@ module ZTK
 ################################################################################
 
     def download(remote, local)
-      @sftp ||= Net::SFTP.start(@config[:host], @config[:ssh_user], ssh_options)
+      @sftp ||= Net::SFTP.start(@config.ssh[:host], @config.ssh[:ssh_user], ssh_options)
 
-      @logger and @logger.debug { "config(#{@config.inspect})" }
-      @logger and @logger.info { "parameters(#{remote},#{local})" }
+      @config.logger and @config.logger.debug { "config(#{@config.ssh.inspect})" }
+      @config.logger and @config.logger.info { "parameters(#{remote},#{local})" }
       @sftp.download!(remote.to_s, local.to_s) do |event, downloader, *args|
         case event
         when :open
-          @logger and @logger.info { "download(#{args[0].remote} -> #{args[0].local})" }
+          @config.logger and @config.logger.info { "download(#{args[0].remote} -> #{args[0].local})" }
         when :close
-          @logger and @logger.debug { "close(#{args[0].local})" }
+          @config.logger and @config.logger.debug { "close(#{args[0].local})" }
         when :mkdir
-          @logger and @logger.debug { "mkdir(#{args[0]})" }
+          @config.logger and @config.logger.debug { "mkdir(#{args[0]})" }
         when :get
-          @logger and @logger.debug { "get(#{args[0].remote}, size #{args[2].size} bytes, offset #{args[1]})" }
+          @config.logger and @config.logger.debug { "get(#{args[0].remote}, size #{args[2].size} bytes, offset #{args[1]})" }
         when :finish
-          @logger and @logger.info { "finish" }
+          @config.logger and @config.logger.info { "finish" }
         end
       end
     end
@@ -147,11 +151,11 @@ module ZTK
 ################################################################################
 
     def proxy_command
-      @logger and @logger.debug { "config(#{@config.inspect})" }
+      @config.logger and @config.logger.debug { "config(#{@config.ssh.inspect})" }
 
-      if !@config[:identity_file]
+      if !@config.ssh[:identity_file]
         message = "You must specify an identity file in order to SSH proxy."
-        @logger and @logger.fatal { message }
+        @config.logger and @config.logger.fatal { message }
         raise SSHError, message
       end
 
@@ -161,25 +165,25 @@ module ZTK
       command << [ "-o", "StrictHostKeyChecking=no" ]
       command << [ "-o", "KeepAlive=yes" ]
       command << [ "-o", "ServerAliveInterval=60" ]
-      command << [ "-i", @config[:proxy_identity_file] ] if @config[:proxy_identity_file]
-      command << "#{@config[:proxy_ssh_user]}@#{@config[:proxy_host]}"
+      command << [ "-i", @config.ssh[:proxy_identity_file] ] if @config.ssh[:proxy_identity_file]
+      command << "#{@config.ssh[:proxy_ssh_user]}@#{@config.ssh[:proxy_host]}"
       command << "nc %h %p"
       command = command.flatten.compact.join(" ")
-      @logger and @logger.debug { "command(#{command})" }
+      @config.logger and @config.logger.debug { "command(#{command})" }
       command
     end
 
 ################################################################################
 
     def ssh_options
-      @logger and @logger.debug { "config(#{@config.inspect})" }
+      @config.logger and @config.logger.debug { "config(#{@config.ssh.inspect})" }
       options = {}
-      options.merge!(:password => @config[:ssh_password]) if @config[:ssh_password]
-      options.merge!(:keys => @config[:identity_file]) if @config[:identity_file]
-      options.merge!(:timeout => @config[:timeout]) if @config[:timeout]
-      options.merge!(:user_known_hosts_file  => '/dev/null') if !@config[:host_key_verify]
-      options.merge!(:proxy => Net::SSH::Proxy::Command.new(proxy_command)) if @config[:proxy]
-      @logger and @logger.debug { "options(#{options.inspect})" }
+      options.merge!(:password => @config.ssh[:ssh_password]) if @config.ssh[:ssh_password]
+      options.merge!(:keys => @config.ssh[:identity_file]) if @config.ssh[:identity_file]
+      options.merge!(:timeout => @config.ssh[:timeout]) if @config.ssh[:timeout]
+      options.merge!(:user_known_hosts_file  => '/dev/null') if !@config.ssh[:host_key_verify]
+      options.merge!(:proxy => Net::SSH::Proxy::Command.new(proxy_command)) if @config.ssh[:proxy]
+      @config.logger and @config.logger.debug { "options(#{options.inspect})" }
       options
     end
 

data/lib/ztk/version.rb

@@ -19,5 +19,5 @@
 ################################################################################
 
 module ZTK
-  VERSION = "0.0.3" unless const_defined?(:VERSION)
+  VERSION = "0.0.4" unless const_defined?(:VERSION)
 end

data/spec/spec_helper.rb

@@ -0,0 +1,38 @@
+################################################################################
+#
+#      Author: Zachary Patten <zachary@jovelabs.com>
+#   Copyright: Copyright (c) Jove Labs
+#     License: Apache License, Version 2.0
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+################################################################################
+
+require "ztk"
+
+################################################################################
+
+$logger = ZTK::Logger.new("/dev/null")
+
+dev_null = File.open("/dev/null", "w")
+$stdout = dev_null
+$stderr = dev_null
+
+################################################################################
+
+require 'simplecov'
+SimpleCov.start do
+  add_filter '/spec/'
+end if ENV["COVERAGE"]
+
+################################################################################

data/spec/ztk/logger_spec.rb

@@ -0,0 +1,133 @@
+################################################################################
+#
+#      Author: Zachary Patten <zachary@jovelabs.com>
+#   Copyright: Copyright (c) Jove Labs
+#     License: Apache License, Version 2.0
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+################################################################################
+
+require "spec_helper"
+
+describe ZTK::Logger do
+
+  before(:all) do
+    ENV["LOG_LEVEL"] = "DEBUG"
+    @messages = {
+      :debug => "This is a test debug message",
+      :info => "This is a test info message",
+      :warn => "This is a test warn message",
+      :error => "This is a test error message",
+      :fatal => "This is a test fatal message"
+    }
+    @logfile = "/tmp/test.log"
+    File.exists?(@logfile) && File.delete(@logfile)
+  end
+
+  subject { ZTK::Logger.new(@logfile) }
+
+  it "should be of kind ZTK::Logger class" do
+    subject.should be_an_instance_of ZTK::Logger
+  end
+
+  describe "general logging functionality" do
+
+    it "should accept debug log messages" do
+      subject.debug { @messages[:debug] }
+      IO.read(@logfile).match(@messages[:debug]).should_not be nil
+    end
+
+    it "should accept info log messages" do
+      subject.info { @messages[:info] }
+      IO.read(@logfile).match(@messages[:info]).should_not be nil
+    end
+
+    it "should accept warn log messages" do
+      subject.warn { @messages[:warn] }
+      IO.read(@logfile).match(@messages[:warn]).should_not be nil
+    end
+
+    it "should accept error log messages" do
+      subject.error { @messages[:error] }
+      IO.read(@logfile).match(@messages[:error]).should_not be nil
+    end
+
+    it "should accept fatal log messages" do
+      subject.fatal { @messages[:fatal] }
+      IO.read(@logfile).match(@messages[:fatal]).should_not be nil
+    end
+
+  end
+
+  describe "log message" do
+
+    before(:all) do
+      subject.debug { @messages[:debug] }
+    end
+
+    it "should contain the date (YYYY-MM-DD)" do
+      IO.read(@logfile).match(Time.now.utc.strftime("%Y-%m-%d")).should_not be nil
+    end
+
+    it "should contain the time (HH:MM)" do
+      IO.read(@logfile).match(Time.now.utc.strftime("%H:%M")).should_not be nil
+    end
+
+    it "should contain the current process ID" do
+      IO.read(@logfile).match(Process.pid.to_s).should_not be nil
+    end
+
+    it "should contain the current log level" do
+      IO.read(@logfile).match("DEBUG").should_not be nil
+    end
+
+    it "should contain the basename of the file containing the method call" do
+      IO.read(@logfile).match(File.basename(__FILE__)).should_not be nil
+    end
+
+    it "should contain the log message itself" do
+      IO.read(@logfile).match(@messages[:debug]).should_not be nil
+    end
+
+  end
+
+  describe "log level" do
+
+    LOG_LEVEL_STEPS = [:debug, :info, :warn, :error, :fatal]
+
+    LOG_LEVEL_STEPS.each do |current_log_level_step|
+
+      it "should allow setting log level to #{current_log_level_step.to_s.upcase} via ENV[\"#{current_log_level_step.to_s.upcase}\"]" do
+
+        ENV["LOG_LEVEL"] = current_log_level_step.to_s.upcase
+        File.exists?(@logfile) && File.delete(@logfile)
+        subject = ZTK::Logger.new(@logfile)
+
+        LOG_LEVEL_STEPS.each do |log_level_step|
+          subject.send(log_level_step) { @messages[log_level_step] }
+          if LOG_LEVEL_STEPS.index(log_level_step) >= LOG_LEVEL_STEPS.index(current_log_level_step)
+            IO.read(@logfile).match(@messages[log_level_step]).should_not be nil
+            IO.read(@logfile).match(log_level_step.to_s.upcase).should_not be nil
+          else
+            IO.read(@logfile).match(@messages[log_level_step]).should be nil
+            IO.read(@logfile).match(log_level_step.to_s.upcase).should be nil
+          end
+        end
+
+      end
+
+    end
+  end
+
+end