syncwrap 1.5.2 → 2.0.0

data/lib/syncwrap/formatter.rb

@@ -0,0 +1,149 @@
+#--
+# Copyright (c) 2011-2014 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'thread'
+require 'term/ansicolor'
+
+module SyncWrap
+
+  class Formatter
+    include Term::ANSIColor
+
+    attr_reader :io
+    attr_reader :lock
+    attr_accessor :colorize
+
+    def initialize( io = $stdout )
+      @io = io
+      @lock = Mutex.new
+      @colorize = true
+      @newlined = true
+      @backtraces = {}
+    end
+
+    def sync( &block )
+      @lock.synchronize( &block )
+    end
+
+    def write_component( host, comp, mth, state )
+      io << yellow if colorize
+      io << '== ' << host.name << ' ' << comp.class << '#' << mth
+      io << ': ' << state
+      io << clear if colorize
+      io << "\n"
+      flush
+    end
+
+    def write_header( host, mode, opts, live = false )
+      olist = []
+      olist << "-#{opts[:sh_verbose]}" if opts[:sh_verbose] && mode != :rsync
+      olist << 'coalesce' if opts[:coalesce]
+      olist << 'dryrun' if opts[:dryrun]
+      olist << "accept:#{opts[:accept].join ','}" if opts[:accept]
+      olist << "user:#{opts[:user]}" if opts[:user]
+      olist << 'live' if live
+
+      io << yellow if colorize
+      io << '<-- ' << mode << ' ' << host.name
+      first = true
+      olist.each do |li|
+        if first
+          io << ' ('
+          first = false
+        else
+          io << ' '
+        end
+        io << li
+      end
+      io << ')' unless olist.empty?
+      io << clear if colorize
+      io << "\n"
+      flush
+    end
+
+    def write_result( result )
+      io << yellow if colorize
+      io << '--> ' << result
+      io << clear if colorize
+      io << "\n"
+      flush
+    end
+
+    def write_command_outputs( outputs, color = true )
+      outputs.each do |stream, buff|
+        write_command_output( stream, buff, color )
+      end
+      output_terminate
+      flush
+    end
+
+    def output_terminate
+      unless @newlined
+        io.puts
+        @newlined = true
+      end
+    end
+
+    def write_command_output( stream, buff, color = true )
+      unless buff.empty?
+        if stream == :err && colorize && color
+          io << red << buff << clear
+        else
+          io << buff
+        end
+        @newlined = ( buff[-1] == "\n" )
+      end
+    end
+
+    def write_error( host, error, comp = nil, mth = nil )
+      bt = error.backtrace
+      bt_num = @backtraces[ bt ]
+      if bt_num
+        bt = nil
+      else
+        @backtraces[ bt ] = bt_num = @backtraces.length + 1
+      end
+
+      io << yellow if colorize
+      io << '== ' << host.name << ' '
+      io << comp.class << '#' << mth << ' ' if comp && mth
+      io << "error"
+      io << ", same stack as" unless bt
+      io << " [" << bt_num << "]:\n"
+
+      io << red if colorize
+      io << short_cn( error.class ) << ': ' << error.message << "\n"
+      if bt
+        bt.each do |line|
+          break if line =~ /execute_component'$/
+          io.puts line
+        end
+      end
+      io << clear if colorize
+      flush
+    end
+
+    def flush
+      io.flush
+    end
+
+    def short_cn( cls )
+      cls.name.sub(/^SyncWrap::/,'')
+    end
+
+  end
+
+end

data/lib/syncwrap/host.rb

@@ -0,0 +1,134 @@
+#--
+# Copyright (c) 2011-2014 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+module SyncWrap
+
+  # Represents various host (server, machine instance) metadata and
+  # serves as a container for #roles and #components.
+  class Host
+
+    # The space in which this host was constructed.
+    attr_reader :space
+
+    # Array of role Symbols or (direct) Component instances in the
+    # order added.
+    attr_reader :contents
+
+    attr_reader :props
+
+    def initialize( space, props = {} )
+      @space = space
+      @props = {}
+      merge_props( props )
+      @contents = [ :all ]
+    end
+
+    # Return the :name property.
+    def name
+      self[ :name ]
+    end
+
+    # Return the property by (Symbol) key
+    def []( key )
+      @props[ key ]
+    end
+
+    # Set property by (Symbol) key to value. Note that the :roles
+    # property key is supported here, but is effectively the same as
+    # #add( val ). Roles are only added, never removed.
+    def []=( key, val )
+      key = key.to_sym
+      if key == :roles
+        add( *val )
+      else
+        @props[ key.to_sym ] = val
+      end
+      val
+    end
+
+    # Merge properties. Note that the :roles property key is
+    # supported here, but is affectively the same as #add( val ).
+    def merge_props( opts )
+      opts.each do |key,val|
+        self[ key ] = val
+      end
+    end
+
+    def to_h
+      @props.merge( roles: roles )
+    end
+
+    # Add any number of roles (by Symbol) or (direct) Component
+    # instances.
+    def add( *args )
+      args.each do |arg|
+        case( arg )
+        when Symbol
+          @contents << arg unless @contents.include?( arg )
+        when Component
+          @contents << arg
+        else
+          raise "Invalid host #{name} addition: #{arg.inspect}"
+        end
+      end
+    end
+
+    # Return an Array of previously added role symbols.
+    def roles
+      @contents.select { |c| c.is_a?( Symbol ) }
+    end
+
+    # Return a flat Array of Component instances by traversing
+    # previously added roles and any direct components in order.
+    def components
+      @contents.inject([]) do |m,c|
+        if c.is_a?( Symbol )
+          m += space.role( c )
+        else
+          m << c
+        end
+        m
+      end
+    end
+
+    # Return the last component added to this Host prior to the given
+    # component (either directly or via a role), or nil if there is no
+    # such component.
+    def prior_component( component )
+      last = nil
+      @contents.each do |c|
+        if c.is_a?( Symbol )
+          space.role( c ).each do |rc|
+            return last if rc.equal?( component ) #identity
+            last = rc
+          end
+        else
+          return last if c.equal?( component ) #identity
+          last = c
+        end
+      end
+      nil
+    end
+
+    # Return the _last_ component which is a kind of the specified
+    # Class or Module clz, or nil if not found.
+    def component( clz )
+      components.reverse.find { |c| c.kind_of?( clz ) }
+    end
+
+  end
+
+end

data/lib/syncwrap/main.rb

@@ -0,0 +1,62 @@
+#--
+# Copyright (c) 2011-2014 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'syncwrap'
+
+module SyncWrap
+
+  # A limited set of (private) methods for use at the top-level in
+  # a sync.rb. All of these methods delegate to the _current_
+  # Space.
+  module Main
+
+    private
+
+    # The current Space
+    def space # :doc:
+      Space.current
+    end
+
+    # Shorthand for space.role
+    def role( *args ) # :doc:
+      space.role( *args )
+    end
+
+    # Shorthand for space.role
+    def host( *args ) # :doc:
+      space.host( *args )
+    end
+
+    # Merge options given, or (without opts) return space.default_options
+    def options( opts = nil ) # :doc:
+      if opts
+        space.merge_default_options( opts )
+      else
+        space.default_options
+      end
+    end
+
+    # Shorthand for space.provider.profile
+    def profile( *args ) # :doc:
+      space.provider.profile( *args )
+    end
+
+  end
+
+end
+
+# Extend the top level Object with the Main module
+self.extend SyncWrap::Main

data/lib/syncwrap/path_util.rb

@@ -0,0 +1,55 @@
+#--
+# Copyright (c) 2011-2014 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'pathname'
+
+module SyncWrap
+
+  # Utility methods for handling paths.
+  module PathUtil
+
+    private
+
+    def caller_path( clr )
+      clr.first =~ /^([^:]+):/ && File.dirname( $1 )
+    end
+
+    # Unless rpath is already absolute, expand it relative to the
+    # calling file, as computed from passed in clr (use your
+    # Kernel#caller)
+    def path_relative_to_caller( rpath, clr ) # :doc:
+      unless rpath =~ %r{^/}
+        from = caller_path( clr )
+        rpath = File.expand_path( rpath, from ) if from
+      end
+      rpath
+    end
+
+    # Return path relative to PWD if the result is shorter, otherwise
+    # return input path. Preserves any trailing '/'.
+    def relativize( path ) # :doc:
+      p = Pathname.new( path )
+      unless p.relative?
+        p = p.relative_path_from( Pathname.pwd ).to_s
+        p += '/' if path[-1] == '/'
+        path = p if p.length < path.length
+      end
+      path
+    end
+
+  end
+
+end

data/lib/syncwrap/rsync.rb

@@ -0,0 +1,227 @@
+#--
+# Copyright (c) 2011-2014 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'tmpdir'
+require 'erb'
+require 'fileutils'
+
+require 'syncwrap/path_util'
+
+module SyncWrap
+
+  # Low level support for rsync command construction and template
+  # processing.
+  module Rsync
+    include PathUtil
+
+    private
+
+    def rsync_args( host, srcs, target, opts = {} ) # :doc:
+
+      # -i --itemize-changes, used for counting changed files
+      flags = %w[ -i ]
+
+      # -r --recursive
+      flags << '-r' unless opts[:recursive] == false
+
+      # -l --links (recreate symlinks on the destination)
+      flags << '-l' unless opts[:links] == false
+
+      # -p --perms (set destination to source permissions)
+      # -E --executability (perserve execute only, default)
+      if opts[:perms] != false
+        if opts[:perms] == :p || opts[:perms].is_a?( String )
+          flags << '-p'
+          if opts[ :perms ].is_a?( String )
+            flags << "--chmod=#{opts[ :perms ]}"
+          end
+        else
+          flags << '-E'
+        end
+      end
+
+      # -c --checksum (to determine changes; not just size,time)
+      flags << '-c' unless opts[:checksum] == false
+
+      # -b --backup (make backups)
+      flags << '-b' unless opts[:backup] == false
+
+      # Pass ssh options via -e (--rsh) flag
+      ssh_flags = []
+      ssh_flags += opts[ :ssh_flags ] if opts[ :ssh_flags ]
+      if opts[ :ssh_options ]
+        ssh_flags += opts[:ssh_options].map { |o| ['-o', o.join('=')] }.flatten
+      end
+      if opts[ :ssh_user ]
+        ssh_flags += [ '-l', opts[ :ssh_user ] ]
+        ssh_flags += [ '-i', opts[ :ssh_user_pem ] ] if opts[ :ssh_user_pem ]
+      end
+      flags += [ '-e', "ssh #{ssh_flags.join ' '}" ] unless ssh_flags.empty?
+
+      if opts[ :user ]
+        # Use sudo to place files at remote.
+        user = opts[ :user ].to_s
+        flags << ( if user != 'root'
+                     "--rsync-path=sudo -u #{user} rsync"
+                   else
+                     "--rsync-path=sudo rsync"
+                   end )
+      end
+
+      excludes = Array( opts[ :excludes ] )
+      flags += excludes.map do |e|
+        if e == :dev
+          '--cvs-exclude'
+        else
+          "--exclude=#{e}"
+        end
+      end
+
+      flags << '-n' if opts[ :dryrun ]
+
+      target = [ host, target ].join(':') unless host == 'localhost'
+
+      [ 'rsync', flags, srcs, target ].flatten.compact
+    end
+
+    def expand_implied_target( srcs ) # :doc:
+      #FIXME: Honor absolute arg paths?
+      if srcs.length == 1
+        target = "/" + srcs.first
+        target = File.dirname( target ) + '/' unless target =~ %r{/$}
+      else
+        target = srcs.pop
+      end
+      [ srcs, target ]
+    end
+
+    # Resolves each srcs path via #resolve_source! Raises SourceNotFound
+    # if any not found.
+    def resolve_sources( srcs, sync_paths ) # :doc:
+      srcs.map { |path| resolve_source!( path, sync_paths ) }
+    end
+
+    # Resolve the specified source path via #resolve_source. Raises
+    # SourceNotFound if not found.
+    def resolve_source!( path, sync_paths ) # :doc:
+      resolve_source( path, sync_paths ) or
+        raise( SourceNotFound,
+              "#{path.inspect} not found in :sync_paths #{sync_paths.inspect}" )
+    end
+
+    # Resolve the specified source path to the first existing
+    # file/directory in sync_paths roots and returns a #relativize
+    # path.  Also tries with an .erb suffix if a src does not have a
+    # trailing '/'. Preserves any trailing '/'. Returns nil if not
+    # found.
+    def resolve_source( path, sync_paths ) # :doc:
+      #FIXME: Honor absolute arg paths?
+      found = nil
+      sync_paths.each do |r|
+        candidate = File.join( r, path )
+        if File.exist?( candidate )
+          found = candidate
+        elsif candidate !~ %r{(\.erb|/)$}
+          candidate += '.erb'
+          if File.exist?( candidate )
+            found = candidate
+          end
+        end
+        break if found
+      end
+
+      found && relativize( found )
+    end
+
+    # Given file path within src, return any sub-directory path needed
+    # to reach file, or the empty string.  This is also src trailing
+    # '/' aware.
+    def subpath( src, file ) # :doc:
+      src = src.sub( %r{/[^/]*$}, '' ) #remove trail slash or last element
+      File.dirname( file ).sub( /^#{src}\/?/, '' )
+    end
+
+    # Process templates in tmpdir and yield post-processed sources to
+    # block, cleaning up on exit.
+    def process_templates( srcs, opts ) # :doc:
+      bnd = opts[ :erb_binding ] or raise "required :erb_binding param missing"
+      erb_mode = opts[ :erb_mode ] || '<>' #Trim new line on "<% ... %>\n"
+      mktmpdir( 'syncwrap' ) do |tmp_dir|
+        processed_sources = []
+        out_dir = File.join( tmp_dir, 'd' ) #for default perms
+        srcs.each do |src|
+          erbs = find_source_erbs( src )
+          outname = nil
+          erbs.each do |erb|
+            spath = subpath( src, erb )
+            outname = File.join( out_dir, spath, File.basename( erb, '.erb' ) )
+            FileUtils.mkdir_p( File.dirname( outname ) )
+            perm = File.stat( erb ).mode
+            File.open( outname, "w", perm ) do |fout|
+              template = ERB.new( IO.read( erb ), nil, erb_mode )
+              template.filename = erb
+              fout.puts( template.result( bnd ) )
+            end
+          end
+          if erbs.length == 1 && src == erbs.first
+            processed_sources << outname
+          elsif !erbs.empty?
+            processed_sources << ( out_dir + '/' )
+          end
+        end
+        yield processed_sources
+      end
+    end
+
+    # Just like Dir.mktmpdir but with an attempt to workaround a JRuby
+    # 1.6.x bug. See https://jira.codehaus.org/browse/JRUBY-5678
+    def mktmpdir( prefix ) # :doc:
+      old_env_tmpdir = nil
+      newdir = nil
+      if defined?( JRUBY_VERSION ) && JRUBY_VERSION =~ /^1.6/
+        old_env_tmpdir = ENV['TMPDIR']
+        newdir = "/tmp/syncwrap.#{ENV['USER']}"
+        FileUtils.mkdir_p( newdir, mode: 0700 )
+        ENV['TMPDIR'] = newdir
+      end
+      Dir.mktmpdir( 'syncwrap' ) do |tmp_dir|
+        yield tmp_dir
+      end
+    ensure
+      FileUtils.rmdir( newdir ) if newdir
+      ENV['TMPDIR'] = old_env_tmpdir if old_env_tmpdir
+    end
+
+    def find_source_erbs( sources ) # :doc:
+      Array( sources ).inject([]) do |list, src|
+        if File.directory?( src )
+          list += find_source_erbs( expand_entries( src ) )
+        elsif src =~ /\.erb$/
+          list << src
+        end
+        list
+      end
+    end
+
+    def expand_entries( src ) # :doc:
+      Dir.entries( src ).
+        reject { |e| e =~ /^\.+$/ }.
+        map { |e| File.join( src, e ) }
+    end
+
+  end
+
+end