syncwrap 1.5.2 → 2.0.0

data/lib/syncwrap/ruby_support.rb

@@ -0,0 +1,110 @@
+#--
+# Copyright (c) 2011-2014 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+module SyncWrap
+
+  # A Support module for Ruby VM components, also providing gem
+  # handling utilties which are largely common to all Ruby VMs.
+  module RubySupport
+
+    # The name of the gem command to be installed/used (default: gem)
+    attr_accessor :gem_command
+
+    # Default gem install arguments (default: --no-rdoc, --no-ri)
+    attr_accessor :gem_install_args
+
+    def initialize( *args )
+      @gem_command = 'gem'
+      @gem_install_args = %w[ --no-rdoc --no-ri ]
+
+      super( *args )
+    end
+
+    def gemrc_path
+      "/etc/gemrc"
+    end
+
+    # Install gemrc file to gemrc_path
+    def install_gemrc
+      rput( 'etc/gemrc', gemrc_path, user: :root )
+    end
+
+    # Install the specified gem.
+    #
+    # === Options
+    #
+    # :version:: Version specifier array or single value, like in a
+    #            gemspec. (Default: nil -> latest) Examples:
+    #
+    #              '1.1.0'
+    #              '~> 1.1'
+    #              ['>=1.0', '<1.2']
+    #
+    # :user_install:: If true, perform a --user-install as the current
+    #                 user.  Otherwise system install with sudo (the
+    #                 default, false).
+    #
+    # :check:: If true, capture output and return the number of gems
+    #          actually installed.  Combine with :minimize to only
+    #          install what is required, and short circuit when zero
+    #          gems installed. (Default: false)
+    #
+    # :minimize:: Use --conservative and --minimal-deps (rubygems
+    #             2.1.5+, #min_deps_supported?) flags to reduce
+    #             installs to the minimum required to satisfy the
+    #             version requirements.  (Default: true)
+    #
+    def gem_install( gem, opts = {} )
+      cmd = [ gem_command, 'install',
+              gem_install_args,
+              ( '--user-install' if opts[ :user_install ] ),
+              ( '--conservative' if opts[ :minimize] != false ),
+              ( '--minimal-deps' if opts[ :minimize] != false &&
+                min_deps_supported? ),
+              gem_version_flags( opts[ :version ] ),
+              gem ].flatten.compact.join( ' ' )
+
+      shopts = opts[ :user_install ] ? {} : {user: :root}
+
+      if opts[ :check ]
+        _,out = capture( cmd, shopts.merge!( accept: 0 ) )
+
+        count = 0
+        out.split( "\n" ).each do |oline|
+          if oline =~ /^\s+(\d+)\s+gem(s)?\s+installed/
+            count = $1.to_i
+          end
+        end
+        count
+      else
+        sh( cmd, shopts )
+      end
+
+    end
+
+    protected
+
+    def min_deps_supported?
+      true
+    end
+
+    def gem_version_flags( reqs )
+      Array( reqs ).flatten.compact.map { |req| "-v'#{req}'" }
+    end
+
+  end
+
+end

data/lib/syncwrap/shell.rb

@@ -0,0 +1,207 @@
+#--
+# Copyright (c) 2011-2014 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#
+# The non-blocking aspects of Shell::capture3 below were partially
+# derived from rake-remote_task, released under MIT License:
+# Copyright (c) Ryan Davis, RubyHitSquad
+#++
+
+require 'open3'
+
+module SyncWrap
+
+  # Low level command construction and process output capture.
+  #
+  # == Supported Commands
+  #
+  # Local:
+  #   bash [-v|-x -e -n] -c COMMANDS
+  #   sudo :sudo_flags [-u user] bash [-v|-x -e -n] -c COMMANDS
+  #
+  # Remote:
+  #   ssh :ssh_flags :host bash [-v|-x -e -n] -c "COMMANDS"
+  #   ssh :ssh_flags :host sudo :sudo_flags [-u user] bash [-v|-x -e -n] -c "COMMANDS"
+  #
+  # Example ssh_flags: -i ./key.pem -l ec2-user
+  #
+  # Example sudo_flags: -H
+  module Shell
+
+    private
+
+    # When host is not 'localhost' return ssh command, flags,
+    # arguments on top of #sudo_args. Otherwise pass through to
+    # #sudo_args.
+    def ssh_args( host, command, opts = {} ) # :doc:
+      args = []
+      if host != 'localhost'
+        opts = opts.dup
+        coalesce = opts.delete( :coalesce )
+        args = [ 'ssh' ]
+        args += opts[ :ssh_flags ] if opts[ :ssh_flags ]
+        if opts[ :ssh_options ]
+          args += opts[ :ssh_options ].map { |o| [ '-o', o.join('=') ] }.flatten
+        end
+        if opts[ :ssh_user ]
+          args += [ '-l', opts[ :ssh_user ] ]
+          args += [ '-i', opts[ :ssh_user_pem ] ] if opts[ :ssh_user_pem ]
+        end
+        args << host.to_s
+        sargs = sudo_args( command, opts )
+        cmd = sargs.pop
+        args += sargs
+        args << ( '"' + shell_escape_cmd( cmd ) + '"' )
+        args << '1>&2' if coalesce
+        args
+      else
+        sudo_args( command, opts )
+      end
+    end
+
+    # Return sudo command, flags, arguments on top of #sh_args if the
+    # :user option is specified. Otherwise pass through to #sh_args.
+    def sudo_args( command, opts = {} ) # :doc:
+      args = []
+      if opts[ :user ]
+        args = [ 'sudo' ]
+        args += opts[ :sudo_flags ] if opts[ :sudo_flags ]
+        # FIXME: Replace with :sudo_home support for '-H'?
+        args += [ '-u', opts[ :user ] ] unless opts[ :user ] == :root
+      end
+      args + sh_args( command, opts )
+    end
+
+    # Return bash command, flags, arguments for the given command(s)
+    # passed to #commmand_lines_cleanup.
+    def sh_args( command, opts = {} ) # :doc:
+      args = [ 'bash' ]
+      args << '-e' if opts[ :error ].nil? || opts[ :error ] == :exit
+      args << '-n' if opts[ :dryrun ]
+
+      if opts[ :coalesce ]
+        args << '-c'
+        cmd = "exec 1>&2\n"
+        if opts[ :sh_verbose ]
+          cmd << "set " << ( opts[ :sh_verbose ] == :x ? '-x' : '-v' ) << "\n"
+        end
+        cmd << command_lines_cleanup( command )
+        args << cmd
+      else
+        if opts[ :sh_verbose ]
+          args << ( opts[ :sh_verbose ] == :x ? '-x' : '-v' )
+        end
+        args << '-c'
+        args << command_lines_cleanup( command )
+      end
+      args
+    end
+
+    # Escape the provided cmd string for inclusion in a bash quoted
+    # string command.  This is only needed when using ssh.
+    def shell_escape_cmd( cmd ) # :doc:
+      cmd.gsub( /["$`\\]/ ) { |c| '\\' + c }
+    end
+
+    # Given one or an Array of commands, apply #block_trim_padding to
+    # each and join all with newlines.
+    def command_lines_cleanup( commands ) # :doc:
+      Array( commands )
+        .map { |cmd| block_trim_padding( cmd.split( $/ ) ) }
+        .flatten
+        .join( "\n" )
+    end
+
+    # Left strip lines, but preserve increased indentation in
+    # subsequent lines. Also right strip and drop blank lines
+    def block_trim_padding( lines ) # :doc:
+      pad = nil
+      lines
+        .reject { |l| l =~ /^\s*$/ } #blank lines
+        .map do |line|
+        line = line.dup
+        unless pad && line.gsub!(/^(\s{,#{pad}})/, '')
+          prior = line.length
+          line.gsub!(/^(\s*)/, '')
+          pad = prior - line.length
+        end
+        line.rstrip!
+        line
+      end
+    end
+
+    # Captures out and err from a command expressed by args
+    # array. Returns [ exit_status, [outputs] ] where [outputs] is an
+    # array of [:err|:out, buffer] elements. Uses select, non-blocking
+    # I/O to receive buffers in the order they become available. This
+    # is often the same order you would see them in a real interactive
+    # terminal, but not always, as buffering or timing issues in the
+    # underlying implementation may cause some out of order results.
+    def capture3( args ) # :doc:
+      status = nil
+      outputs = []
+      Open3::popen3( *args ) do |inp, out, err, wait_thread|
+        inp.close rescue nil
+
+        streams = [ err, out ]
+
+        until streams.empty? do
+          selected, = select( streams, nil, nil, 0.1 )
+          next if selected.nil? || selected.empty?
+
+          selected.each do |stream|
+            if stream.eof?
+              streams.delete( stream )
+              next
+            end
+
+            chunk = stream.readpartial( 8192 )
+            marker = (stream == out) ? :out : :err
+
+            yield( marker, chunk ) if block_given?
+
+            # Merge chunks from the same stream
+            l = outputs.last
+            if l && l[0] == marker
+              l[1] += chunk
+            else
+              outputs << [ marker, chunk ]
+            end
+
+          end
+        end
+
+        # Older jruby (even in 1.9+ mode) doesn't provide wait_thread but
+        # does return the status in $? instead (see workaround below)
+        status = wait_thread.value if wait_thread
+      end
+
+      #FIXME: Only if jruby?
+      status ||= $?
+
+      [ status && status.exitstatus, outputs ]
+    end
+
+    # Select and merge the output buffers of the specific stream from
+    # outputs (as returned by #capture3)
+    def collect_stream( stream, outputs ) # :doc:
+      outputs.
+        select { |o| o[0] == stream }.
+        map { |o| o[1] }. #the buffers
+        inject( "", :+ )
+    end
+
+  end
+
+end

data/lib/syncwrap.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2011-2013 David Kellum
+# Copyright (c) 2011-2014 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may
@@ -14,4 +14,370 @@
 # permissions and limitations under the License.
 #++
 
+require 'thread'
+
 require 'syncwrap/base'
+require 'syncwrap/component'
+require 'syncwrap/context'
+require 'syncwrap/host'
+require 'syncwrap/formatter'
+require 'syncwrap/path_util'
+
+module SyncWrap
+
+  # Base class for all SyncWrap exception types
+  class SyncError < RuntimeError
+  end
+
+  # Error in the context in which a component is used
+  class ContextError < SyncError
+  end
+
+  # Error in use of the Context#sh block form (:close option) with
+  # a nested flush or change in options.
+  class NestingError < SyncError
+  end
+
+  # Source specified in rput can not be found in :sync_paths
+  class SourceNotFound < SyncError
+  end
+
+  # Context#sh or derivatives failed with non-accepted exit code.
+  # Note the error may be delayed until the next Context#flush.
+  class CommandFailure < SyncError
+  end
+
+  # Serves as the container for #hosts and roles and provides the top
+  # level #execute.
+  class Space
+    include PathUtil
+
+    # Return the current space, as setup within a Space#with block, or
+    # raise something fierce.
+    def self.current
+      Thread.current[:syncwrap_current_space] or
+        raise "Space.current called outside of Space#with/thread"
+    end
+
+    # Default options, for use including Component#rput, Component#sh,
+    # and #execute (see Options details). The CLI uses this, for
+    # example, to set :verbose => true (from --verbose) and
+    # :shell_verbose => :x (from --expand-shell). In limited cases it
+    # may be appropriate to set default overrides in a sync.rb.
+    attr_reader :default_options
+
+    attr_reader :formatter #:nodoc:
+
+    def initialize
+      @provider = nil
+      @roles = Hash.new { |h,k| h[k] = [] }
+      @hosts = {}
+      @default_options = {
+        coalesce: true,
+        sh_verbose: :v,
+        sync_paths: [ File.join( SyncWrap::GEM_ROOT, 'sync' ) ] }
+      @formatter = Formatter.new
+    end
+
+    # Load the specified file path as per a sync.rb, into this
+    # Space. If relative, path is assumed to be relative to the caller
+    # (i.e. Rakefile, etc.) as with the conventional 'sync' directory.
+    def load_sync_file_relative( fpath = './sync.rb' )
+      load_sync_file( path_relative_to_caller( fpath, caller ) )
+    end
+
+    # Load the specified filename as per a sync.rb, into this Space.
+    # This uses a #with block internally.
+    def load_sync_file( filename )
+      require 'syncwrap/main'
+      with do
+        load( filename, true )
+        # This is true -> wrapped to avoid pollution of sync
+        # namespace. This is particularly important given the dynamic
+        # binding scheme of components. If not done, top-level
+        # methods/vars in sync.rb would have precidents over
+        # component methods.
+      end
+    end
+
+    # Make self the Space.current inside of block.
+    def with
+      prior = Thread.current[:syncwrap_current_space]
+      raise "Invalid Space#with nesting!" if prior && prior != self
+      begin
+        Thread.current[:syncwrap_current_space] = self
+        yield self
+      ensure
+        Thread.current[:syncwrap_current_space] = prior
+      end
+    end
+
+    # A hosting/cloud provider for creating/removing hosts from this
+    # space. See #use_provider
+    def provider
+      @provider or raise "No provider set via space.use_provider"
+    end
+
+    # Merge the specified options to default_options
+    def merge_default_options( opts )
+      @default_options.merge!( opts )
+      nil
+    end
+
+    # Prepend the given directory path to front of the :sync_paths
+    # list. If relative, path is assumed to be relative to the caller
+    # (i.e. sync.rb) as with the conventional 'sync'
+    # directory. Returns a copy of the resultant sync_paths list.
+    def prepend_sync_path( rpath = 'sync' )
+      rpath = path_relative_to_caller( rpath, caller )
+
+      roots = default_options[ :sync_paths ]
+      roots.delete( rpath ) # don't duplicate but move to front
+      roots.unshift( rpath )
+      roots.dup #return a copy
+    end
+
+    # Create a new instance of the specified provider class for use,
+    # passing self and the given options.
+    def use_provider( provider_class, opts = {} )
+      opts = opts.merge( sync_file_path: caller_path( caller ) )
+      @provider = provider_class.new( self, opts )
+    end
+
+    # Define/access a Role by symbol.
+    # Additional args are interpreted as Components to add to this
+    # role.
+    def role( symbol, *args )
+      if args.empty?
+        @roles[ symbol.to_sym ]
+      else
+        @roles[ symbol.to_sym ] += args.flatten.compact
+      end
+    end
+
+    # Define/access a Host by name.
+    #
+    # If first arg is a String, it is interpreted as the name
+    # property.  The name property is also used for lookup and thus
+    # must be Space unique. Additional args are interpreted as role
+    # symbols or (direct) Components to add to this Host. Each role
+    # will only be added once. A final Hash argument is interpreted as
+    # properties to add or merge with the host.
+    def host( *args )
+      props = args.last.is_a?( Hash ) && args.pop || {}
+      name = args.first.is_a?( String ) && args.shift
+      props = props.merge( name: name ) if name
+      raise "Missing required name parameter" unless props[ :name ]
+      host = @hosts[ props[ :name ] ] ||= Host.new( self )
+      host.merge_props( props )
+      host.add( *args )
+      host
+    end
+
+    # All Host instances, in order added.
+    def hosts
+      @hosts.values
+    end
+
+    def host_names
+      @hosts.keys
+    end
+
+    # Return an ordered, unique set of component classes, direct or via
+    # roles, currently contained by the specified hosts or all hosts.
+    def component_classes( hs = hosts )
+      hs.
+        map { |h| h.components }.
+        flatten.
+        map { |comp| comp.class }.
+        uniq
+    end
+
+    # Returns a new component_plan from plan, looking up any Class
+    # string names and using :install for any missing methods:
+    #
+    # \[ [ Class | String, Symbol? ] ... ] -> [ [ Class, Symbol ] ... ]
+    #
+    # Class name lookup is by unqualified matching against
+    # #component_classes (already added to hosts of this space.) If
+    # such String match is ambiguous or not found, a RuntimeError is
+    # raised.
+    def resolve_component_plan( plan )
+      classes = component_classes
+      plan.map do |clz,mth|
+        if clz.is_a?( String )
+          found = classes.select { |cc| cc.name =~ /(^|::)#{clz}$/ }
+          if found.length == 1
+            clz = found.first
+          else
+            raise "Class \"#{clz}\" ambiguous or not found: #{found.inspect}"
+          end
+        end
+        [ clz, mth && mth.to_sym || :install ]
+      end
+    end
+
+    # Execute components based on a host_list (default all), a
+    # component_plan (default :install on all components), and with
+    # any additional options (merged with default_options).
+    #
+    # === Options
+    #
+    # The following options are specifically handled by execute:
+    #
+    # :colorize:: If false, don't color diagnostic output to stdout
+    #             (default: true)
+    #
+    # :threads:: The number of threads on which to execute. Each host is
+    #            executed with a single thread.
+    #            (Default: the number of hosts, maximum concurrency)
+    #
+    def execute( host_list = hosts, component_plan = [], opts = {} )
+      opts = default_options.merge( opts )
+      @formatter.colorize = ( opts[ :colorize ] != false )
+      component_plan = resolve_component_plan( component_plan )
+
+      if opts[ :threads ] && host_list.length > opts[ :threads ]
+        queue = Queue.new
+        host_list.each { |host| queue.push( host ) }
+        threads = opts[ :threads ].times.map do
+          Thread.new( queue, component_plan, opts ) do |q, cp, o|
+            success = true
+            begin
+              while host = q.pop( true ) # non-block
+                r = execute_host( host, cp, o )
+                success &&= r
+              end
+            rescue ThreadError
+              #exit, from queue being empty
+            end
+            success
+          end
+        end
+      else
+        threads = host_list.map do |host|
+          Thread.new( host, component_plan, opts ) do |h, cp, o|
+            execute_host( h, cp, o )
+          end
+        end
+      end
+      threads.inject(true) { |s,t| t.value && s }
+      # Note: Unhandled (i.e. non-SyncError) exceptions will be
+      # propigated and re-raised on call to value above, resulting in
+      # standard ruby stack trace and immediate exit.
+    end
+
+    # Given a Host, determine the address to use for ssh (incl. rsync)
+    # access. The following properties are used in order of decreasing
+    # preference: internet_name, internet_ip and host.name.
+    def ssh_host_name( host )
+      # This is included here for expected Space-wide policy settings.
+      host[ :internet_name ] || host[ :internet_ip ] || host.name
+    end
+
+    private
+
+    def execute_host( host, component_plan = [], opts = {} )
+      # Important: resolve outside of context
+      comp_methods = resolve_component_methods(host.components, component_plan)
+      ctx = Context.new( host, opts )
+      ctx.with do
+        comp_methods.each do |comp, mths|
+          success = mths.inject(true) do |s, mth|
+            # short-circuit after first non-success
+            s && execute_component( ctx, host, comp, mth, opts )
+          end
+          return false unless success
+        end
+      end
+      true
+    rescue SyncError => e
+      formatter.sync do
+        formatter.write_error( host, e )
+      end
+      false
+    end
+
+    # Given components and plan, return an ordered Array of
+    # \[component, [method,...]] to execute. An empty/default plan is
+    # interpreted as :install on all components which implement it. If
+    # :install is explicitly part of the plan, then it trumps any
+    # other methods listed for the same component.
+    #
+    # Note this must be run out-of-Context to avoid unintended dynamic
+    # binding of :install or other methods.
+    def resolve_component_methods( components, component_plan = [] )
+      components.map do |comp|
+        mths = []
+        if component_plan.empty?
+          mths = [ :install ] if comp.respond_to?( :install )
+        else
+          found = component_plan.select { |cls,_| comp.kind_of?( cls ) }
+          mths = found.map { |_,mth| mth }
+          mths = [ :install ] if mths.include?( :install ) #trumps
+        end
+        [ comp, mths ] unless mths.empty?
+      end.compact
+    end
+
+    def execute_component( ctx, host, comp, mth, opts )
+
+      formatter.sync do
+        formatter.write_component( host, comp, mth,
+          opts[ :flush_component ] ? "start" : "enqueue" )
+      end
+
+      comp.send( mth )
+
+      ctx.flush if opts[ :flush_component ]
+
+      if opts[ :flush_component ]
+        formatter.sync do
+          formatter.write_component( host, comp, mth, "complete" )
+        end
+      end
+
+      true
+    rescue SyncError => e
+      formatter.sync do
+        if e.is_a?( CommandFailure ) && ! opts[ :flush_component ]
+          # We don't know if its really from this component/method
+          formatter.write_error( host, e )
+        else
+          formatter.write_error( host, e, comp, mth )
+        end
+      end
+      false
+    end
+
+  end
+
+  # Setup autoloads for all included components. This should not be
+  # risky in SyncWrap given that all of these should be loaded before
+  # any threads are in play.
+
+  autoload :CommercialJDK, 'syncwrap/components/commercial_jdk'
+  autoload :CRubyVM,       'syncwrap/components/cruby_vm'
+  autoload :EtcHosts,      'syncwrap/components/etc_hosts'
+  autoload :Geminabox,     'syncwrap/components/geminabox'
+  autoload :Hashdot,       'syncwrap/components/hashdot'
+  autoload :Iyyov,         'syncwrap/components/iyyov'
+  autoload :IyyovDaemon,   'syncwrap/components/iyyov_daemon'
+  autoload :JRubyVM,       'syncwrap/components/jruby_vm'
+  autoload :MDRaid,        'syncwrap/components/mdraid'
+  autoload :Network,       'syncwrap/components/network'
+  autoload :OpenJDK,       'syncwrap/components/open_jdk'
+  autoload :PostgreSQL,    'syncwrap/components/postgresql'
+  autoload :Qpid,          'syncwrap/components/qpid'
+  autoload :QpidRepo,      'syncwrap/components/qpid'
+  autoload :RHEL,          'syncwrap/components/rhel'
+  autoload :RunUser,       'syncwrap/components/run_user'
+  autoload :Ubuntu,        'syncwrap/components/ubuntu'
+  autoload :Users,         'syncwrap/components/users'
+
+  # (Alpha sort class name)
+
+  # Additional autoloads (optional support)
+  autoload :AmazonEC2,     'syncwrap/amazon_ec2'
+
+end

data/{etc → sync/etc}/gemrc

@@ -2,10 +2,8 @@
 install: --no-ri --no-rdoc
 update:  --no-ri --no-rdoc
 :backtrace: false
-:benchmark: false
-:bulk_threshold: 1000
 :verbose: true
 :update_sources: true
 :sources:
-- http://rubygems.org/
+- https://rubygems.org/
 ...