syncwrap 1.5.2 → 2.0.0

data/lib/syncwrap/components/users.rb

@@ -0,0 +1,200 @@
+#--
+# Copyright (c) 2011-2014 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'syncwrap/component'
+require 'syncwrap/path_util'
+
+module SyncWrap
+
+  # Provision developer user accounts, sudo access, and synchronize
+  # home directory files.
+  class Users < Component
+    include PathUtil
+
+    # The list of user names to install. If default nil, home_users
+    # will be determined by the set of home directories found in
+    # local_home_root - exclude_users.
+    attr_accessor :home_users
+
+    # Local home root directory, to be resolved against sync_paths.
+    # (Default: home)
+    attr_accessor :local_home_root
+
+    # Set of users to exclude from synchronization (default: [])
+    attr_accessor :exclude_users
+
+    # If set, override the :ssh_user for this Component install, since
+    # typically the 'normal' user (i.e your developer account) has not
+    # yet been created or given sudo access.  (Default: nil)
+    attr_accessor :ssh_user
+
+    # A PEM file for the ssh_user. A relative path in interpreted as
+    # relative to the sync file from which this component is
+    # created. If the pem file is not found, a warning will be issued
+    # and ssh_user and ssh_user_pem will not be used.
+    # (Default: nil)
+    attr_accessor :ssh_user_pem
+
+    # Users should be the first component to attempt ssh access. On
+    # new host creation, install may be run before the ssh port is
+    # actually available (boot completed, etc.). This provides an
+    # additional timeout in seconds for establishing the first ssh
+    # session. (default: 180 seconds)
+    attr_accessor :ssh_access_timeout
+
+    # Use the StrictHostKeyChecking=no ssh option when connected to a
+    # newly created host (whose key will surely not be known yet.)
+    # (Default: true)
+    attr_accessor :lenient_host_key
+
+    def initialize( opts = {} )
+      @home_users = nil
+      @local_home_root = "home"
+      @exclude_users = []
+      @ssh_user = nil
+      @ssh_user_pem = nil
+      @ssh_access_timeout = 180.0
+      @lenient_host_key = true
+      super
+
+      if @ssh_user_pem
+        @ssh_user_pem =
+          relativize( path_relative_to_caller( @ssh_user_pem, caller ) )
+        unless File.exist?( @ssh_user_pem )
+          warn( "WARNING: Users pem #{@ssh_user_pem} not found. " +
+                "Will not use #{@ssh_user}.\n" +
+                "Components may fail without sudo access" )
+          @ssh_user = nil
+          @ssh_user_pem = nil
+        end
+      end
+    end
+
+    def install
+      ensure_ssh_access if host[ :just_created ] && ssh_access_timeout > 0
+
+      rdir = find_source( local_home_root )
+      users = home_users
+      users ||= rdir && Dir.entries( rdir ).select do |d|
+        ( d !~ /^\./ && File.directory?( File.join( rdir, d ) ) )
+      end
+      users ||= []
+      users -= exclude_users
+
+      users.each do |u|
+        create_user( u )
+        set_sudoers( u )
+      end
+
+      users.each do |u|
+        sync_home_files( u )
+      end
+
+      users.each do |u|
+        fix_home_permissions( u )
+      end
+
+      #FIXME: Add special case for 'root' user?
+    end
+
+    def ensure_ssh_access
+      flags = ssh_flags
+      if lenient_host_key
+        flags[ :ssh_options ] ||= {}
+        flags[ :ssh_options ][ 'StrictHostKeyChecking' ] = 'no'
+      end
+
+      start = Time.now
+      loop do
+        accept  = [0]
+        # Allow non-sucess until timeout
+        # 255: ssh (i.e. can't connect, sshd not yet up)
+        # 1: sudo error (user_data sudoers update has not yet completed)
+        accept += [1, 255] unless ( Time.now - start ) >= ssh_access_timeout
+
+        code,_ = capture( 'sudo true', flags.merge( accept: accept ) )
+        break if code == 0
+        sleep 1 # ssh timeouts also apply, but make sure we don't spin
+      end
+    end
+
+    # Create user if not already present
+    def create_user( user )
+      sudo <<-SH
+        if ! id #{user} >/dev/null 2>&1; then
+          useradd #{user}
+        fi
+      SH
+    end
+
+    def sync_home_files( user )
+      rput( "#{local_home_root}/#{user}", user: user )
+    rescue SourceNotFound
+      false
+    end
+
+    def fix_home_permissions( user )
+      sudo <<-SH
+        if [ -e '/home/#{user}/.ssh' ]; then
+          chmod 700 /home/#{user}/.ssh
+          chmod -R o-rwx /home/#{user}/.ssh
+        fi
+      SH
+    end
+
+    def set_sudoers( user )
+      #FIXME: make this a template?
+      #FIXME: Use local_root for secure_path? (Order issue)
+      #Relax, less overrides needed for Ubuntu?
+      sudo <<-SH
+        echo '#{user} ALL=(ALL) NOPASSWD:ALL'  > /etc/sudoers.d/#{user}
+        echo 'Defaults:#{user} !requiretty'   >> /etc/sudoers.d/#{user}
+        echo 'Defaults:#{user} secure_path = /sbin:/bin:/usr/sbin:/usr/bin:/usr/local/bin' \
+          >> /etc/sudoers.d/#{user}
+        chmod 440 /etc/sudoers.d/#{user}
+      SH
+
+      #FIXME: Centos 6.5:
+      # secure_path is the same as above already
+      #FIXME: echo 'Defaults:#{user} always_set_home' >> /etc/sudoers.d/#{user}
+    end
+
+    protected
+
+    def sh( command, opts = {}, &block )
+      super( command, ssh_flags.merge( opts ), &block )
+    end
+
+    def rput( *args )
+      opts = args.last.is_a?( Hash ) ? ssh_flags.merge( args.pop ) : ssh_flags
+      super( *args, opts )
+    end
+
+    def ssh_flags
+      flags = {}
+      if ssh_user
+        flags[ :ssh_user ] = ssh_user
+        if ssh_user_pem
+          flags[ :ssh_user_pem ] = ssh_user_pem
+          flags[ :ssh_options ] = { 'IdentitiesOnly' => 'yes' }
+        end
+      end
+      flags
+    end
+
+  end
+
+end

data/lib/syncwrap/context.rb

@@ -0,0 +1,260 @@
+#--
+# Copyright (c) 2011-2014 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'syncwrap/shell'
+require 'syncwrap/rsync'
+
+module SyncWrap
+
+  # A single-thread execution context for a single Host.
+  #
+  # Context implements much of the interface and behavior defined by
+  # Component, via use of the Shell and Rsync module mixins.
+  class Context
+    include Shell
+    include Rsync
+
+    class << self
+      # Return the current (thread local) Context, or nil.
+      def current
+        Thread.current[:syncwrap_context]
+      end
+
+      # Set the current Context to ctx and return any prior Context or
+      # nil.
+      def swap( ctx )
+        old = current
+        Thread.current[:syncwrap_context] = ctx
+        old
+      end
+    end
+
+    # The current Host of this context
+    attr_reader :host
+
+    # Construct given host and default_options to use for all #sh and
+    # #rput calls.
+    def initialize( host, opts = {} )
+      @host = host
+      reset_queue
+      @queue_locked = false
+      @default_options = opts
+
+      super()
+    end
+
+    # Set (thread local) current context to self, yield to block, then
+    # #flush and reset the context.
+    def with
+      prior = Context.swap( self )
+      yield
+      flush
+    ensure
+      Context.swap( prior )
+    end
+
+    # Return true if being executed, by constructed default options,
+    # in dryrun mode.
+    def dryrun?
+      @default_options[ :dryrun ]
+    end
+
+    # Return true if :verbose is set in constructed default options.
+    def verbose?
+      @default_options[ :verbose ]
+    end
+
+    # See Component#sh for interface details
+    def sh( command, opts = {} )
+      opts = @default_options.merge( opts )
+      close = opts.delete( :close )
+
+      flush if opts != @queued_opts #may still be a no-op
+
+      @queued_cmd << command
+      @queued_opts = opts
+
+      if close
+        prev, @queue_locked = @queue_locked, true
+      end
+
+      begin
+        yield if block_given?
+        @queued_cmd << close if close
+      ensure
+        @queue_locked = prev if close
+      end
+      nil
+    end
+
+    # See Component#flush for interface details
+    def flush
+      if @queued_cmd.length > 0
+        begin
+          if @queue_locked
+            raise NestingError, 'Queue at flush: ' + @queued_cmd.join( '\n' )
+          end
+          run_shell( @queued_cmd, @queued_opts )
+        ensure
+          reset_queue
+        end
+      end
+      nil
+    end
+
+    # See Component#capture for interface details.
+    def capture( command, opts = {} )
+      opts = @default_options.merge( coalesce: false, dryrun: false ).merge( opts )
+      flush
+      capture_shell( command, opts )
+    end
+
+    # See Component#rput for interface details.
+    def rput( *args )
+      opts = @default_options
+      opts = opts.merge( args.pop ) if args.last.is_a?( Hash )
+      opts = opts.merge( coalesce: false )
+
+      flush
+
+      srcs, target = expand_implied_target( args )
+
+      srcs = resolve_sources( srcs, Array( opts[ :sync_paths ] ) )
+
+      changes = []
+
+      if opts[:erb_process] != false
+        sdirs, sfiles =   srcs.partition { |src| File.directory?( src ) }
+        serbs, sfiles = sfiles.partition { |src| src =~ /\.erb$/ }
+        plains = sdirs + sfiles #might not have/is not templates
+        maybes = sdirs + serbs  #might have/is templates
+
+        if maybes.empty?
+          changes = rsync( plains, target, opts ) unless plains.empty?
+        else
+          process_templates( maybes, opts ) do |processed|
+            unless processed.empty? || plains.empty?
+              opts = opts.dup
+              opts[ :excludes ] = Array( opts[ :excludes ] ) + [ '*.erb' ]
+            end
+            new_srcs = plains + processed
+            changes = rsync( new_srcs, target, opts ) unless new_srcs.empty?
+          end
+        end
+      else
+        changes = rsync( srcs, target, opts ) unless srcs.empty?
+      end
+
+      changes
+    end
+
+    # Returns the path to the the specified src, first found in
+    # :sync_paths option.  Returns nil if not found.
+    def find_source( src, opts = {} )
+      opts = @default_options.merge( opts )
+      resolve_source( src, Array( opts[ :sync_paths ] ) )
+    end
+
+    private
+
+    def ssh_host_name
+      host.space.ssh_host_name( host )
+    end
+
+    def reset_queue
+      @queued_cmd = []
+      @queued_opts = {}
+    end
+
+    def run_shell( command, opts = {} )
+      args = ssh_args( ssh_host_name, command, opts )
+      capture_stream( args, host, :sh, opts )
+    end
+
+    def capture_shell( command, opts = {} )
+      args = ssh_args( ssh_host_name, command, opts )
+      exit_code, outputs = capture_stream( args, host, :capture, opts )
+      [ exit_code, collect_stream( opts[ :coalesce ] ? :err : :out, outputs ) ]
+    end
+
+    def rsync( srcs, target, opts )
+      args = rsync_args( ssh_host_name, srcs, target, opts )
+      exit_code, outputs = capture_stream( args, host, :rsync, opts )
+
+      # Return array of --itemize-changes on standard out.
+      collect_stream( :out, outputs ).
+        split( "\n" ).
+        map { |l| l =~ /^(\S{11})\s(.+)$/ && [$1, $2] }. #itemize-changes
+        compact
+    end
+
+    def capture_stream( args, host, mode, opts )
+      accept = opts[ :accept ] || [ 0 ]
+      success_msg = ( accept == [ 0 ] ) ? "success" : "accepted"
+
+      # When :verbose -> nil -> try_lock
+      stream_output = opts[ :verbose ] ? nil : false
+      fmt = host.space.formatter
+      do_color = !opts[ :coalesce ]
+
+      begin
+        exit_code, outputs = capture3( args ) do |stream, chunk|
+          if stream_output.nil?
+            if fmt.lock.try_lock
+              stream_output = true
+              fmt.write_header( host, mode, opts, :stream )
+              if mode == :rsync
+                cmd = args.join(' ') + "\n"
+                fmt.write_command_output( :cmd, cmd, do_color )
+              end
+            else
+              stream_output = false
+            end
+          end
+
+          if stream_output
+            fmt.write_command_output( stream, chunk, do_color )
+            fmt.flush
+          end
+        end
+        failed = !accept.include?( exit_code )
+
+        if stream_output
+          fmt.output_terminate
+          fmt.write_result( "Exit #{exit_code} (#{success_msg})" ) unless failed
+        end
+      ensure
+        fmt.lock.unlock if stream_output
+      end
+
+      if !stream_output && ( failed || opts[ :verbose ] )
+        fmt.sync do
+          fmt.write_header( host, mode, opts )
+          if mode == :rsync
+            fmt.write_command_output( :cmd, args.join(' ') + "\n", do_color )
+          end
+          fmt.write_command_outputs( outputs, do_color )
+          fmt.write_result( "Exit #{exit_code} (#{success_msg})" ) unless failed
+        end
+      end
+
+      raise CommandFailure, "#{args[0]} exit code: #{exit_code}" if failed
+      [ exit_code, outputs ]
+    end
+
+  end
+
+end

data/lib/syncwrap/distro.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2011-2013 David Kellum
+# Copyright (c) 2011-2014 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may
@@ -16,77 +16,70 @@
 
 require 'syncwrap/base'
 
-# Support for distro specializations.
-module SyncWrap::Distro
+module SyncWrap
 
-  # A Hash of internal/common package names to distro specific package
-  # names.  Here defined as empty.
-  attr_reader :packages_map
+  # Support module and inteface for (Linux) distribution components.
+  module Distro
 
-  def initialize
-    super
+    # The root directory for local, non-distro managed installs
+    # (default: /usr/local)
+    attr_accessor :local_root
 
-    @packages_map = {}
-  end
+    # A Hash of internal/common package names to distro specific package
+    # names.
+    attr_reader :packages_map
 
-  # Map internal/common names and return distro-specific names. If a
-  # mapping does not exist, assume internal name is a common name.
-  def dist_map_packages( *pkgs )
-    pkgs.flatten.compact.map { |pkg| packages_map[ pkg ] || pkg }
-  end
+    def initialize( *args )
+      @local_root = '/usr/local'
+      @packages_map = {}
 
-  # Install the specified packages using distro-specific mapped
-  # package names and command via dist_install_s. The last argument is
-  # interpreted as options if it is a Hash.
-  # === Options
-  # :succeed:: Always succeed (useful for local rpms which might
-  # already be installed.
-  # :minimal:: Avoid additional "optional" packages when possible.
-  def dist_install( *args )
-    sudo( dist_install_s( *args ) )
-  end
+      super( *args )
+    end
 
-  # Generate command to install the specified packages using
-  # distro-specific mapped package names. The last argument is
-  # interpreted as options if it is a Hash.
-  def dist_install_s( *args )
-    raise "Include a distro-specific module, e.g. Ubuntu, RHEL"
-  end
+    # Return self, and in Context, the specific Distro component.
+    def distro
+      self
+    end
 
-  # Uninstall specified packages using distro-specific mapped
-  # package names and command.
-  def dist_uninstall( *args )
-    sudo( dist_uninstall_s( *args ) )
-  end
+    # Map internal/common names and return distro-specific names. If a
+    # mapping does not exist, return the original name.
+    def dist_map_packages( *pkgs )
+      pkgs.flatten.compact.map { |pkg| packages_map[ pkg ] || pkg }
+    end
 
-  # Return command to uninstall specified packages using
-  # distro-specific mapped package names.
-  def dist_uninstall_s( *args )
-    raise "Include a distro-specific module, e.g. Ubuntu, RHEL"
-  end
+    # Install the specified packages using distro-specific mapped
+    # package names. A trailing hash is interpreted as options, see
+    # below.
+    #
+    # ==== Options
+    #
+    # :succeed:: Always succeed (useful for local rpms which might
+    #            already be installed.
+    #
+    # :minimal:: Avoid additional "optional" packages when possible.
+    def dist_install( *pkgs )
+      raise "Include a distro-specific component, e.g. Ubuntu, RHEL"
+    end
 
-  # Install a System V style init.d script (already placed at remote
-  # /etc/init.d/<name>) via distro-specific command
-  def dist_install_init_service( name )
-    sudo( dist_install_init_service_s( name ) )
-  end
+    # Uninstall specified packages using distro-specific mapped
+    # package names and command.
+    def dist_uninstall( *pkgs )
+      raise "Include a distro-specific component, e.g. Ubuntu, RHEL"
+    end
 
-  # Return command to install a System V style init.d script (already
-  # placed at remote /etc/init.d/<name>).
-  def dist_install_init_service_s( name )
-    raise "Include a distro-specific module, e.g. Ubuntu, RHEL"
-  end
+    # Install a System V style init.d script (already placed at remote
+    # /etc/init.d/<name>) via distro-specific command
+    def dist_install_init_service( name )
+      raise "Include a distro-specific component, e.g. Ubuntu, RHEL"
+    end
 
-  # Run via sudo, the System V style, distro specific `service`
-  # command, typically supporting 'start', 'stop', 'restart',
-  # 'status', etc. arguments.
-  def dist_service( *args )
-    sudo( dist_service_s( *args ) )
-  end
+    # Run via sudo, the System V style, distro specific `service`
+    # command, typically supporting 'start', 'stop', 'restart',
+    # 'status', etc. arguments.
+    def dist_service( *args )
+      raise "Include a distro-specific component, e.g. Ubuntu, RHEL"
+    end
 
-  # Return the System V style, distro specific `service` command.
-  def dist_service_s( *args )
-    raise "Include a distro-specific module, e.g. Ubuntu, RHEL"
   end
 
 end