ztk 0.0.2 → 0.0.3

data/.gitignore

@@ -1,3 +1,4 @@
+*.log
 *.gem
 *.rbc
 .bundle

data/README.md

@@ -18,9 +18,6 @@ Or install it yourself as:
 
 ## Usage
 
-TODO: Write usage instructions here
-
-
 ### Parallel
 
 Parallel Processing Class
@@ -38,3 +35,9 @@ Example:
     parallel.waitall
     parallel.results
     => [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
+
+### Logger
+
+Logging Class
+
+This is a logging class based off the ruby core Logger class; but with very verbose logging information, adding PID, micro second timing to log messages.  It favors passing messages via blocks in order to speed up execution when log messages do not need to be yielded.

data/bin/ztk

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+require "pry"
+require "ztk"
+
+##
+#
+# Welcome to the ZTK pry shell!
+#
+##
+binding.pry

data/lib/ztk.rb

@@ -1,8 +1,29 @@
-require "ztk/version"
+################################################################################
+#
+#      Author: Zachary Patten <zachary@jovelabs.com>
+#   Copyright: Copyright (c) Jove Labs
+#     License: Apache License, Version 2.0
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+################################################################################
 
-require "ztk/logger"
-require "ztk/parallel"
+require "ztk/version"
 
 module ZTK
-  # Your code goes here...
+  class Error < StandardError; end
+
+  autoload :Logger, "ztk/logger"
+  autoload :Parallel, "ztk/parallel"
+  autoload :SSH, "ztk/ssh"
 end

data/lib/ztk/logger.rb

@@ -1,42 +1,79 @@
+################################################################################
+#
+#      Author: Zachary Patten <zachary@jovelabs.com>
+#   Copyright: Copyright (c) Jove Labs
+#     License: Apache License, Version 2.0
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+################################################################################
+
 require "logger"
 
-class ZTK::Logger < ::Logger
-  SEVERITIES = Severity.constants.inject([]) {|arr,c| arr[Severity.const_get(c)] = c; arr}
+module ZTK
+  class Logger < ::Logger
 
-  def initialize(filename)
-    super(filename)
-    set_log_level
-  end
+################################################################################
+
+    SEVERITIES = Severity.constants.inject([]) {|arr,c| arr[Severity.const_get(c)] = c; arr}
 
-  def parse_caller(at)
-    if /^(.+?):(\d+)(?::in `(.*)')?/ =~ at
-      file = Regexp.last_match[1]
-      line = Regexp.last_match[2]
-      method = Regexp.last_match[3]
-      "#{File.basename(file)}:#{line}:#{method} | "
-    else
-      ""
+################################################################################
+
+    def initialize(*args)
+      super(*args)
+      set_log_level
     end
-  end
 
-  def add(severity, message = nil, progname = nil, &block)
-    return if (@level > severity)
+################################################################################
 
-    called_by = parse_caller(caller[1])
-    msg = (block && block.call)
-    return if (msg.nil? || msg.strip.empty?)
-    message = [message, progname, msg].delete_if{|i| i == nil}.join(": ")
-    message = "%19s.%06d+%05d|%5s|%s%s\n" % [Time.now.utc.strftime("%Y-%m-%d %H:%M:%S"), Time.now.utc.usec, Process.pid, SEVERITIES[severity], called_by, message]
+    def parse_caller(at)
+      if /^(.+?):(\d+)(?::in `(.*)')?/ =~ at
+        file = Regexp.last_match[1]
+        line = Regexp.last_match[2]
+        method = Regexp.last_match[3]
+        "#{File.basename(file)}:#{line}:#{method} | "
+      else
+        ""
+      end
+    end
 
-    @logdev.write(message)
+################################################################################
 
-    true
-  end
+    def add(severity, message = nil, progname = nil, &block)
+      return if (@level > severity)
 
-  def set_log_level(level=nil)
-    defined?(Rails) and (default = (Rails.env.production? ? "INFO" : "DEBUG")) or (default = "INFO")
-    log_level = (ENV['LOG_LEVEL'] || level || default)
-    self.level = ZTK::Logger.const_get(log_level.to_s.upcase)
-  end
+      called_by = parse_caller(caller[1])
+      msg = (block && block.call)
+      return if (msg.nil? || msg.strip.empty?)
+      message = [message, progname, msg].delete_if{|i| i == nil}.join(": ")
+      message = "%19s.%06d+%05d|%5s|%s%s\n" % [Time.now.utc.strftime("%Y-%m-%d %H:%M:%S"), Time.now.utc.usec, Process.pid, SEVERITIES[severity], called_by, message]
+
+      @logdev.write(message)
 
+      true
+    end
+
+################################################################################
+
+    def set_log_level(level=nil)
+      defined?(Rails) and (default = (Rails.env.production? ? "INFO" : "DEBUG")) or (default = "INFO")
+      log_level = (ENV['LOG_LEVEL'] || level || default)
+      self.level = ZTK::Logger.const_get(log_level.to_s.upcase)
+    end
+
+################################################################################
+
+  end
 end
+
+################################################################################

data/lib/ztk/parallel.rb

@@ -1,81 +1,120 @@
-class ZTK::Parallel
-  attr_accessor :results
-
-  def initialize(options={})
-    GC.copy_on_write_friendly = true if GC.respond_to?(:copy_on_write_friendly=)
+################################################################################
+#
+#      Author: Zachary Patten <zachary@jovelabs.com>
+#   Copyright: Copyright (c) Jove Labs
+#     License: Apache License, Version 2.0
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+################################################################################
+
+module ZTK
+  class Parallel
+
+################################################################################
+
+    attr_accessor :results
+
+################################################################################
+
+    def initialize(options={})
+      GC.copy_on_write_friendly = true if GC.respond_to?(:copy_on_write_friendly=)
+
+      options.reverse_merge!(
+        :max_forks => `grep -c processor /proc/cpuinfo`.strip.to_i,
+        :one_shot => false
+      )
+
+      @max_forks = options[:max_forks]
+      @one_shot = options[:one_shot]
+
+      @forks = Array.new
+      @results = Array.new
+    end
 
-    options.reverse_merge!(
-      :max_forks => `grep -c processor /proc/cpuinfo`.strip.to_i,
-      :one_shot => false
-    )
+################################################################################
 
-    @max_forks = options[:max_forks]
-    @one_shot = options[:one_shot]
+    def process
+      pid = nil
+      return pid if (@forks.count >= @max_forks)
 
-    @forks = Array.new
-    @results = Array.new
-  end
+      child_reader, parent_writer = IO.pipe
+      parent_reader, child_writer = IO.pipe
 
-  def process
-    pid = nil
-    return pid if (@forks.count >= @max_forks)
+      ActiveRecord::Base.connection.disconnect!
+      pid = Process.fork do
+        ActiveRecord::Base.establish_connection
 
-    child_reader, parent_writer = IO.pipe
-    parent_reader, child_writer = IO.pipe
-
-    ActiveRecord::Base.connection.disconnect!
-    pid = Process.fork do
-      ActiveRecord::Base.establish_connection
+        parent_writer.close
+        parent_reader.close
 
-      parent_writer.close
-      parent_reader.close
+        if (data = yield).present?
+          child_writer.write(Base64.encode64(Marshal.dump(data)))
+        end
 
-      if (data = yield).present?
-        child_writer.write(Base64.encode64(Marshal.dump(data)))
+        child_reader.close
+        child_writer.close
+        Process.exit!(0)
       end
+      ActiveRecord::Base.establish_connection
 
       child_reader.close
       child_writer.close
-      Process.exit!(0)
+
+      fork = {:reader => parent_reader, :writer => parent_writer, :pid => pid}
+      @forks << fork
+
+      pid
     end
-    ActiveRecord::Base.establish_connection
 
-    child_reader.close
-    child_writer.close
+################################################################################
 
-    fork = {:reader => parent_reader, :writer => parent_writer, :pid => pid}
-    @forks << fork
+    def wait
+      pid, status = (Process.wait2(-1, Process::WNOHANG) rescue nil)
+      if pid.present? && status.present?
+        if (fork = @forks.select{ |f| f[:pid] == pid }.first).present?
+          data = (Marshal.load(Base64.decode64(fork[:reader].read.to_s)) rescue nil)
+          @results.push(data) if (data.present? && !@one_shot)
 
-    pid
-  end
+          fork[:reader].close
+          fork[:writer].close
 
-  def wait
-    pid, status = (Process.wait2(-1, Process::WNOHANG) rescue nil)
-    if pid.present? && status.present?
-      if (fork = @forks.select{ |f| f[:pid] == pid }.first).present?
-        data = (Marshal.load(Base64.decode64(fork[:reader].read.to_s)) rescue nil)
-        @results.push(data) if (data.present? && !@one_shot)
+          @forks -= [fork]
+          return [pid, status, data]
+        end
+      end
+      nil
+    end
 
-        fork[:reader].close
-        fork[:writer].close
+################################################################################
 
-        @forks -= [fork]
-        return [pid, status, data]
+    def waitall
+      results = Array.new
+      while @forks.count > 0
+        results << wait
       end
+      results
     end
-    nil
-  end
 
-  def waitall
-    results = Array.new
-    while @forks.count > 0
-      results << wait
+################################################################################
+
+    def count
+      @forks.count
     end
-    results
-  end
 
-  def count
-    @forks.count
-  end
+################################################################################
 
+  end
 end
+
+################################################################################

data/lib/ztk/ssh.rb

@@ -0,0 +1,191 @@
+################################################################################
+#
+#      Author: Zachary Patten <zachary@jovelabs.com>
+#   Copyright: Copyright (c) Jove Labs
+#     License: Apache License, Version 2.0
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+################################################################################
+
+module ZTK
+  class SSHError < Error; end
+  class SSH
+
+################################################################################
+
+    attr_accessor :stdout, :stderr, :stdin, :config
+
+################################################################################
+
+    def initialize(logger=$logger, stdout=$stdout, stderr=$stderr, stdin=$stdin)
+      @logger, @stdout, @stderr, @stdin = logger, stdout, stderr, stdin
+
+      @logger.sync = true if @logger.respond_to?(:sync=)
+      @stdout.sync = true if @stdout.respond_to?(:sync=)
+      @stderr.sync = true if @stderr.respond_to?(:sync=)
+      @stdin.sync = true if @stdin.respond_to?(:sync=)
+
+      @config = Hash.new(nil)
+    end
+
+################################################################################
+
+    def console
+      @logger and @logger.debug { "config(#{@config.inspect})" }
+
+      command = [ "ssh" ]
+      command << [ "-q" ]
+      command << [ "-o", "UserKnownHostsFile=/dev/null" ]
+      command << [ "-o", "StrictHostKeyChecking=no" ]
+      command << [ "-o", "KeepAlive=yes" ]
+      command << [ "-o", "ServerAliveInterval=60" ]
+      command << [ "-i", @config[:identity_file] ] if @config[:identity_file]
+      command << [ "-o", "ProxyCommand=\"#{proxy_command}\"" ] if @config[:proxy]
+      command << "#{@config[:ssh_user]}@#{@config[:host]}"
+      command = command.flatten.compact.join(" ")
+      @logger and @logger.info { "command(#{command})" }
+      Kernel.exec(command)
+    end
+
+################################################################################
+
+    def exec(command, options={})
+      @ssh ||= Net::SSH.start(@config[:host], @config[:ssh_user], ssh_options)
+
+      options = { :silence => false }.merge(options)
+      silence = options[:silence]
+      output = ""
+
+      @logger and @logger.debug { "config(#{@config.inspect})" }
+      @logger and @logger.debug { "options(#{options.inspect})" }
+      @logger and @logger.info { "command(#{command})" }
+      channel = @ssh.open_channel do |chan|
+        @logger and @logger.debug { "channel opened" }
+        chan.exec(command) do |ch, success|
+          raise SSHError, "Could not execute '#{command}'." unless success
+
+          ch.on_data do |c, data|
+            output += data
+            @logger and @logger.debug { data.chomp.strip }
+            @stdout.print(data) if !silence
+          end
+
+          ch.on_extended_data do |c, type, data|
+            output += data
+            @logger and @logger.debug { data.chomp.strip }
+            @stderr.print(data) if !silence
+          end
+
+        end
+      end
+      channel.wait
+      @logger and @logger.debug { "channel closed" }
+
+      output
+    end
+
+################################################################################
+
+    def upload(local, remote)
+      @sftp ||= Net::SFTP.start(@config[:host], @config[:ssh_user], ssh_options)
+
+      @logger and @logger.debug { "config(#{@config.inspect})" }
+      @logger and @logger.info { "parameters(#{local},#{remote})" }
+      @sftp.upload!(local.to_s, remote.to_s) do |event, uploader, *args|
+        case event
+        when :open
+          @logger and @logger.info { "upload(#{args[0].local} -> #{args[0].remote})" }
+        when :close
+          @logger and @logger.debug { "close(#{args[0].remote})" }
+        when :mkdir
+          @logger and @logger.debug { "mkdir(#{args[0]})" }
+        when :put
+          @logger and @logger.debug { "put(#{args[0].remote}, size #{args[2].size} bytes, offset #{args[1]})" }
+        when :finish
+          @logger and @logger.info { "finish" }
+        end
+      end
+    end
+
+################################################################################
+
+    def download(remote, local)
+      @sftp ||= Net::SFTP.start(@config[:host], @config[:ssh_user], ssh_options)
+
+      @logger and @logger.debug { "config(#{@config.inspect})" }
+      @logger and @logger.info { "parameters(#{remote},#{local})" }
+      @sftp.download!(remote.to_s, local.to_s) do |event, downloader, *args|
+        case event
+        when :open
+          @logger and @logger.info { "download(#{args[0].remote} -> #{args[0].local})" }
+        when :close
+          @logger and @logger.debug { "close(#{args[0].local})" }
+        when :mkdir
+          @logger and @logger.debug { "mkdir(#{args[0]})" }
+        when :get
+          @logger and @logger.debug { "get(#{args[0].remote}, size #{args[2].size} bytes, offset #{args[1]})" }
+        when :finish
+          @logger and @logger.info { "finish" }
+        end
+      end
+    end
+
+
+################################################################################
+  private
+################################################################################
+
+    def proxy_command
+      @logger and @logger.debug { "config(#{@config.inspect})" }
+
+      if !@config[:identity_file]
+        message = "You must specify an identity file in order to SSH proxy."
+        @logger and @logger.fatal { message }
+        raise SSHError, message
+      end
+
+      command = ["ssh"]
+      command << [ "-q" ]
+      command << [ "-o", "UserKnownHostsFile=/dev/null" ]
+      command << [ "-o", "StrictHostKeyChecking=no" ]
+      command << [ "-o", "KeepAlive=yes" ]
+      command << [ "-o", "ServerAliveInterval=60" ]
+      command << [ "-i", @config[:proxy_identity_file] ] if @config[:proxy_identity_file]
+      command << "#{@config[:proxy_ssh_user]}@#{@config[:proxy_host]}"
+      command << "nc %h %p"
+      command = command.flatten.compact.join(" ")
+      @logger and @logger.debug { "command(#{command})" }
+      command
+    end
+
+################################################################################
+
+    def ssh_options
+      @logger and @logger.debug { "config(#{@config.inspect})" }
+      options = {}
+      options.merge!(:password => @config[:ssh_password]) if @config[:ssh_password]
+      options.merge!(:keys => @config[:identity_file]) if @config[:identity_file]
+      options.merge!(:timeout => @config[:timeout]) if @config[:timeout]
+      options.merge!(:user_known_hosts_file  => '/dev/null') if !@config[:host_key_verify]
+      options.merge!(:proxy => Net::SSH::Proxy::Command.new(proxy_command)) if @config[:proxy]
+      @logger and @logger.debug { "options(#{options.inspect})" }
+      options
+    end
+
+################################################################################
+
+  end
+end
+
+################################################################################

data/lib/ztk/version.rb

@@ -1,3 +1,23 @@
+################################################################################
+#
+#      Author: Zachary Patten <zachary@jovelabs.com>
+#   Copyright: Copyright (c) Jove Labs
+#     License: Apache License, Version 2.0
+#
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+#
+#       http://www.apache.org/licenses/LICENSE-2.0
+#
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+#
+################################################################################
+
 module ZTK
-  VERSION = "0.0.2" unless const_defined?(:VERSION)
+  VERSION = "0.0.3" unless const_defined?(:VERSION)
 end

data/ztk.gemspec

@@ -6,7 +6,7 @@ Gem::Specification.new do |gem|
   gem.email         = ["zachary@jovelabs.com"]
   gem.description   = %q{Zachary's Toolkit}
   gem.summary       = %q{Contains various helper classes and utility methods I find I regularly need.}
-  gem.homepage      = "http://www.github.com/jovelabs"
+  gem.homepage      = "https://github.com/jovelabs/ztk"
 
   gem.files         = `git ls-files`.split($\)
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
@@ -14,4 +14,8 @@ Gem::Specification.new do |gem|
   gem.name          = "ztk"
   gem.require_paths = ["lib"]
   gem.version       = ZTK::VERSION
+
+  gem.add_dependency("net-ssh")
+  gem.add_dependency("net-sftp")
+  gem.add_development_dependency("pry")
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ztk
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
   prerelease: 
 platform: ruby
 authors:
@@ -9,12 +9,61 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-24 00:00:00.000000000 Z
-dependencies: []
+date: 2012-08-31 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: net-ssh
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: net-sftp
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Zachary's Toolkit
 email:
 - zachary@jovelabs.com
-executables: []
+executables:
+- ztk
 extensions: []
 extra_rdoc_files: []
 files:
@@ -24,12 +73,14 @@ files:
 - LICENSE
 - README.md
 - Rakefile
+- bin/ztk
 - lib/ztk.rb
 - lib/ztk/logger.rb
 - lib/ztk/parallel.rb
+- lib/ztk/ssh.rb
 - lib/ztk/version.rb
 - ztk.gemspec
-homepage: http://www.github.com/jovelabs
+homepage: https://github.com/jovelabs/ztk
 licenses: []
 post_install_message: 
 rdoc_options: []