syncwrap 2.6.2 → 2.7.0

data/lib/syncwrap/components/lvm_cache.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2011-2015 David Kellum
+# Copyright (c) 2011-2016 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/lib/syncwrap/components/mdraid.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2011-2015 David Kellum
+# Copyright (c) 2011-2016 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/lib/syncwrap/components/network.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2011-2015 David Kellum
+# Copyright (c) 2011-2016 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/lib/syncwrap/components/open_jdk.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2011-2015 David Kellum
+# Copyright (c) 2011-2016 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/lib/syncwrap/components/postgresql.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2011-2015 David Kellum
+# Copyright (c) 2011-2016 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may
@@ -40,6 +40,7 @@ module SyncWrap
   # * RHEL, CentOS 7: 9.2
   # * AmazonLinux 2013.03: 8.4 9.2
   # * AmazonLinux 2014.09: 8.4 9.2 9.3
+  # * AmazonLinux 2015.09: 9.2 9.3 9.4
   # * Debian 7: 9.1
   # * Debian 8: 9.4
   # * Ubuntu 14: 9.3
@@ -150,8 +151,25 @@ module SyncWrap
     # increased risk. (PG Default: 0 -> none)
     attr_accessor :commit_delay
 
-    # WAL log segments (16MB each) (PG Default: 3)
-    attr_accessor :checkpoint_segments
+    # WAL log segments (16MB each)
+    # Deprecated with PostgreSQL 9.5: Use min/max_wal_size instead
+    attr_writer :checkpoint_segments
+
+    def checkpoint_segments
+      @checkpoint_segments || ( version_lt?(pg_version, [9,5]) ? 3 : 5 )
+    end
+
+    # Minimum WAL size as string with units
+    # Default: PG <9.5: "48MB"; PG 9.5+: "80MB"
+    attr_writer :min_wal_size
+
+    def min_wal_size
+      @min_wal_size || "#{ checkpoint_segments * 16 }MB"
+    end
+
+    # Maximum WAL size as string with units.
+    # (Default: unset, PG Default: '1GB')
+    attr_accessor :max_wal_size
 
     # Shared buffers (Default: '256MB' vs PG: '128MB')
     attr_accessor :shared_buffers
@@ -231,7 +249,9 @@ module SyncWrap
       @service_name = 'postgresql'
       @synchronous_commit = :on
       @commit_delay = 0
-      @checkpoint_segments = 3
+      @checkpoint_segments = nil
+      @min_wal_size = nil
+      @max_wal_size = nil
       @shared_buffers = '256MB'
       @work_mem = '128MB'
       @maintenance_work_mem = '128MB'

data/lib/syncwrap/components/puma.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2011-2015 David Kellum
+# Copyright (c) 2011-2016 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/lib/syncwrap/components/qpid.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2011-2015 David Kellum
+# Copyright (c) 2011-2016 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/lib/syncwrap/components/rake_gem.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2011-2015 David Kellum
+# Copyright (c) 2011-2016 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/lib/syncwrap/components/rhel.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2011-2015 David Kellum
+# Copyright (c) 2011-2016 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may
@@ -45,17 +45,21 @@ module SyncWrap
     # interpreted as options, see below.
     #
     # ==== Options
-    # :succeed:: Always succeed (useful for local rpm files which
-    #            might already be installed.)
     #
-    # Other options will be ignored.
+    # :check_install:: Short-circuit if all packages already
+    #                  installed. Thus no upgrades will be performed.
+    #
+    # :succeed:: Deprecated, use check_install instead
+    #
+    # Additional options are passed to the sudo calls.
     def dist_install( *pkgs )
-      opts = pkgs.last.is_a?( Hash ) && pkgs.pop || {}
-
-      if opts[ :succeed ]
-        sudo "yum install -q -y #{pkgs.join( ' ' )} || true"
-      else
-        sudo "yum install -q -y #{pkgs.join( ' ' )}"
+      opts = pkgs.last.is_a?( Hash ) && pkgs.pop.dup || {}
+      opts.delete( :minimal )
+      pkgs.flatten!
+      chk = opts.delete( :check_install ) || opts.delete( :succeed )
+      chk = check_install? if chk.nil?
+      dist_if_not_installed?( pkgs, chk, opts ) do
+        sudo( "yum install -q -y #{pkgs.join( ' ' )}", opts )
       end
     end
 
@@ -63,19 +67,36 @@ module SyncWrap
     # interpreted as options, see below.
     #
     # ==== Options
-    # :succeed:: Succeed even if no such packages are installed
     #
-    # Other options will be ignored.
+    # :succeed:: Succeed even if none of the packages are
+    #            installed. (Deprecated, Default: true)
+    #
+    # Additional options are passed to the sudo calls.
     def dist_uninstall( *pkgs )
-      opts = pkgs.last.is_a?( Hash ) && pkgs.pop || {}
-      if opts[ :succeed ]
-        sudo <<-SH
-          if yum list -q installed #{pkgs.join( ' ' )} >/dev/null 2>&1; then
+      opts = pkgs.last.is_a?( Hash ) && pkgs.pop.dup || {}
+      pkgs.flatten!
+      if opts.delete( :succeed ) != false
+        sudo( <<-SH, opts )
+          if yum list -C -q installed #{pkgs.join( ' ' )} >/dev/null 2>&1; then
             yum remove -q -y #{pkgs.join( ' ' )}
           fi
         SH
       else
-        sudo "yum remove -q -y #{pkgs.join( ' ' )}"
+        sudo( "yum remove -q -y #{pkgs.join( ' ' )}", opts )
+      end
+    end
+
+    # If chk is true, then wrap block in a sudo bash conditional
+    # testing if any specified pkgs are not installed. Otherwise just
+    # yield to block.
+    def dist_if_not_installed?( pkgs, chk, opts, &block )
+      if chk
+        qry = "yum list -C -q installed #{pkgs.join ' '}"
+        cnt = qry + " | tail -n +2 | wc -l"
+        cond = %Q{if [ "$(#{cnt})" != "#{pkgs.count}" ]; then}
+        sudo( cond, opts.merge( close: 'fi' ), &block )
+      else
+        block.call
       end
     end
 

data/lib/syncwrap/components/run_user.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2011-2015 David Kellum
+# Copyright (c) 2011-2016 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/lib/syncwrap/components/source_tree.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2011-2015 David Kellum
+# Copyright (c) 2011-2016 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/lib/syncwrap/components/tarpit_gem.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2011-2015 David Kellum
+# Copyright (c) 2011-2016 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/lib/syncwrap/components/ubuntu.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2011-2015 David Kellum
+# Copyright (c) 2011-2016 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/lib/syncwrap/components/users.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2011-2015 David Kellum
+# Copyright (c) 2011-2016 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may
@@ -99,6 +99,15 @@ module SyncWrap
         set_sudoers( u )
       end
 
+      # Some distro's, like Debian, don't come with rsync installed so
+      # need to install it here. For backward compatibly, only do
+      # this if dist_install is defined (i.e. Distro component before
+      # self.)
+      if !users.empty? && respond_to?( :dist_install )
+        dist_install( 'rsync',
+                      ssh_flags.merge( minimal: true, check_install: true ) )
+      end
+
       users.each do |u|
         sync_home_files( u )
       end

data/lib/syncwrap/context.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2011-2015 David Kellum
+# Copyright (c) 2011-2016 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You
@@ -82,6 +82,12 @@ module SyncWrap
       @default_options[ :verbose ]
     end
 
+    # Return any value of :check_install set in constructed default
+    # options.
+    def check_install?
+      @default_options[ :check_install ]
+    end
+
     # See Component#sh for interface details
     def sh( command, opts = {} )
       opts = @default_options.merge( opts )
@@ -150,6 +156,10 @@ module SyncWrap
         if maybes.empty?
           changes = rsync( plains, target, opts ) unless plains.empty?
         else
+          if ssh_host_name == 'localhost' && opts[ :user ]
+            # tmpdir needs to be visable to alt. opts[ :user ]
+            opts[ :tmpdir_mode ] = 0755
+          end
           process_templates( maybes, opts ) do |processed|
             unless processed.empty? || plains.empty?
               opts = opts.dup

data/lib/syncwrap/distro.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2011-2015 David Kellum
+# Copyright (c) 2011-2016 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may
@@ -41,15 +41,20 @@ module SyncWrap
     #
     # ==== Options
     #
-    # :succeed:: Always succeed (useful for local rpms which might
-    #            already be installed.
+    # :check_install:: Short-circuit if all packages already
+    #                  installed. Thus no upgrades will be performed.
     #
-    # :minimal:: Avoid additional "optional" packages when possible.
+    # :succeed:: Deprecated, use check_install instead
+    #
+    # :minimal:: Avoid additional "optional" packages when possible
+    #
+    # Additional options are passed to the sudo calls.
     def dist_install( *pkgs )
       raise "Include a distro-specific component, e.g. Debian, RHEL"
     end
 
-    # Uninstall the specified package names.
+    # Uninstall the specified package names. A trailing hash is
+    # interpreted as options, passed to the sudo calls.
     def dist_uninstall( *pkgs )
       raise "Include a distro-specific component, e.g. Debian, RHEL"
     end

data/lib/syncwrap/formatter.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2011-2015 David Kellum
+# Copyright (c) 2011-2016 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/lib/syncwrap/git_help.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2011-2015 David Kellum
+# Copyright (c) 2011-2016 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/lib/syncwrap/hash_support.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2011-2015 David Kellum
+# Copyright (c) 2011-2016 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/lib/syncwrap/host.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2011-2015 David Kellum
+# Copyright (c) 2011-2016 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You

data/lib/syncwrap/main.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2011-2015 David Kellum
+# Copyright (c) 2011-2016 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/lib/syncwrap/path_util.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2011-2015 David Kellum
+# Copyright (c) 2011-2016 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/lib/syncwrap/rsync.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2011-2015 David Kellum
+# Copyright (c) 2011-2016 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may
@@ -169,7 +169,7 @@ module SyncWrap
     def process_templates( srcs, opts ) # :doc:
       bnd = opts[ :erb_binding ] or raise "required :erb_binding param missing"
       erb_mode = opts[ :erb_mode ] || '<>' #Trim new line on "<% ... %>\n"
-      mktmpdir( 'syncwrap-' ) do |tmp_dir|
+      mktmpdir( opts ) do |tmp_dir|
         processed_sources = []
         out_dir = File.join( tmp_dir, 'd' ) #for default perms
         srcs.each do |src|
@@ -196,23 +196,14 @@ module SyncWrap
       end
     end
 
-    # Just like Dir.mktmpdir but with an attempt to workaround a JRuby
-    # 1.6.x bug. See https://jira.codehaus.org/browse/JRUBY-5678
-    def mktmpdir( prefix ) # :doc:
-      old_env_tmpdir = nil
-      newdir = nil
-      if defined?( JRUBY_VERSION ) && JRUBY_VERSION =~ /^1.6/
-        old_env_tmpdir = ENV['TMPDIR']
-        newdir = "/tmp/syncwrap.#{ENV['USER']}"
-        FileUtils.mkdir_p( newdir, mode: 0700 )
-        ENV['TMPDIR'] = newdir
-      end
-      Dir.mktmpdir( prefix ) do |tmp_dir|
-        yield tmp_dir
+    # Like Dir.mktmpdir but with option to specify :tmpdir_mode.
+    def mktmpdir( opts ) # :doc:
+      path = Dir::Tmpname.create( 'syncwrap-' ) do |n|
+        Dir.mkdir( n, opts[ :tmpdir_mode ] || 0700 )
       end
+      yield path
     ensure
-      FileUtils.rmdir( newdir ) if newdir
-      ENV['TMPDIR'] = old_env_tmpdir if old_env_tmpdir
+      FileUtils.remove_entry( path ) if path
     end
 
     def find_source_erbs( sources ) # :doc:

data/lib/syncwrap/ruby_support.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2011-2015 David Kellum
+# Copyright (c) 2011-2016 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/lib/syncwrap/shell.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2011-2015 David Kellum
+# Copyright (c) 2011-2016 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/lib/syncwrap/systemd.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2011-2015 David Kellum
+# Copyright (c) 2011-2016 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/lib/syncwrap/user_data.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2011-2015 David Kellum
+# Copyright (c) 2011-2016 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/lib/syncwrap/version_support.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2011-2015 David Kellum
+# Copyright (c) 2011-2016 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may

data/lib/syncwrap/zone_balancer.rb

@@ -0,0 +1,65 @@
+#--
+# Copyright (c) 2011-2016 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+module SyncWrap
+
+  # Utility for balancing new hosts accross multiple (AWS)
+  # availability zones for fault tolarance.
+  module ZoneBalancer
+
+    # Returns a ruby Proc which when called will return the best
+    # pick of availability zone, via ::next_zone. This variant
+    # uses Space.current within a Space#with block.
+    def self.zone( zones, roles )
+      space = Space.current
+      lambda do
+        next_zone( space, zones, roles )
+      end
+    end
+
+    # Return the next best zone from zones Array<String>, preferring
+    # the least frequent :availability_zone of existing hosts in the
+    # specified space and roles (Array<Symbol>, if empty all hosts).
+    def self.next_zone( space, zones, roles = [] )
+      if zones
+        hosts = filter_hosts( space.hosts, roles )
+        zfreqs = {}
+        zones.each { |z| zfreqs[z] = 0 }
+        czones = hosts.map { |h| h[:availability_zone] }.compact
+        czones.each { |z| zfreqs[z] += 1 if zfreqs.has_key?( z ) }
+
+        # Sort by ascending frequency (lowest first). Keep order stable
+        # from original zones, when frequency tied.
+        # Return the first (least frequent, zones stable) zone.
+        n = 0
+        zfreqs.sort_by { |_,f| [ f, (n+=1) ] }.first[0]
+      end
+    end
+
+    private
+
+    def self.filter_hosts( hosts, roles )
+      unless roles.empty?
+        hosts = hosts.select do |h|
+          h.roles.any? { |r| roles.include?( r ) }
+        end
+      end
+      hosts
+    end
+
+  end
+
+end

data/lib/syncwrap.rb

@@ -1,5 +1,5 @@
 #--
-# Copyright (c) 2011-2015 David Kellum
+# Copyright (c) 2011-2016 David Kellum
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you
 # may not use this file except in compliance with the License.  You may
@@ -22,6 +22,7 @@ require 'syncwrap/context'
 require 'syncwrap/host'
 require 'syncwrap/formatter'
 require 'syncwrap/path_util'
+require 'syncwrap/version_support.rb'
 
 module SyncWrap
 
@@ -51,6 +52,7 @@ module SyncWrap
   # level #execute.
   class Space
     include PathUtil
+    include VersionSupport
 
     # Return the current space, as setup within a Space#with block, or
     # raise something fierce.
@@ -92,11 +94,11 @@ module SyncWrap
     def load_sync_file( filename )
       require 'syncwrap/main'
       with do
-        load( filename, true )
-        # This is true -> wrapped to avoid pollution of sync
-        # namespace. This is particularly important given the dynamic
-        # binding scheme of components. If not done, top-level
-        # methods/vars in sync.rb would have precidents over
+        load( filename, wrap_sync_load? )
+        # Should wrap to avoid pollution of sync namespace, but there
+        # are jruby bugs to workaround. This is particularly important
+        # given the dynamic binding scheme of components. If not done,
+        # top-level methods/vars in sync.rb would have precidents over
         # component methods.
       end
     end
@@ -291,6 +293,16 @@ module SyncWrap
 
     private
 
+    # Return true if wrapped load should be attempted on this ruby
+    # Avoid this only on JRuby 9.0.0-9.0.4, which will fail hard.
+    #
+    # See https://github.com/jruby/jruby/issues/3180
+    def wrap_sync_load?
+      ( !defined?( JRUBY_VERSION ) ||
+        version_lt?(  JRUBY_VERSION, [9] ) ||
+        version_gte?( JRUBY_VERSION, [9,0,5] ) )
+    end
+
     def execute_host( host, component_plan = [], opts = {} )
       comp_roles = opts[ :comp_roles ]
       comps = if comp_roles && !comp_roles.empty?
@@ -413,5 +425,6 @@ module SyncWrap
   autoload :AmazonEC2,     'syncwrap/amazon_ec2'
   autoload :GitHelp,       'syncwrap/git_help'
   autoload :UserData,      'syncwrap/user_data'
+  autoload :ZoneBalancer,  'syncwrap/zone_balancer'
 
 end