syncwrap 2.5.1 → 2.6.0

data/lib/syncwrap/components/lvm_cache.rb

@@ -0,0 +1,116 @@
+#--
+# Copyright (c) 2011-2015 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'syncwrap/component'
+
+module SyncWrap
+
+  # Provision a cache using a fast drive (like an SSD) over a
+  # slow drive (magnetic or network attached) via LVM's cache-pool
+  # mechanism and dm-cache.
+  #
+  # Host component dependencies: <Distro>
+  class LVMCache < Component
+
+    protected
+
+    # The target size of the cache metadata volume as a string. The
+    # dm-cache guidelines are 1/1000th of the cache volume size and a
+    # minimum of 8M (MiB). Default: '8M'
+    attr_accessor :meta_size
+
+    # The target size of the cache volume as a string. This may either
+    # express a percentage (via -l) or a real size (via -L, ex: '30G')
+    # Default: '100%FREE'
+    attr_accessor :cache_size
+
+    # A volume group instance number compatible with the naming used
+    # by MDRaid. The cache and cache-meta volumes must all be under
+    # the same volume group as the lv_cache_target volume.
+    # Default: 0 -> 'vg0'
+    attr_accessor :vg_instance
+
+    # The fast raw device to use as the cache, i.e. '/dev/xvdb'
+    # (required)
+    attr_accessor :raw_device
+
+    # The target volume name in the same volume group to
+    # cache. (required)
+    attr_accessor :lv_cache_target
+
+    # The name of the cache volume
+    # Default: lv_cache_target + '_cache'
+    attr_writer :lv_cache
+
+    # Array of additional flags to vgextend
+    # Default: ['-y']
+    attr_accessor :vgextend_flags
+
+    def lv_cache
+      @lv_cache || ( lv_cache_target + '_cache' )
+    end
+
+    def lv_cache_meta
+      lv_cache + '_meta'
+    end
+
+    def vg
+      "vg#{vg_instance}"
+    end
+
+    public
+
+    def initialize( opts = {} )
+      @meta_size = '8M'
+      @cache_size = '100%FREE'
+      @vg_instance = 0
+      @raw_device = nil
+      @lv_cache_target = nil
+      @lv_cache = nil
+      @vgextend_flags = %w[ -y ]
+      super
+
+      raise "LVMCache#raw_device not set" unless raw_device
+      raise "LVMCache#lv_cache_target not set" unless lv_cache_target
+    end
+
+    def install
+      dist_install( "lvm2", minimal: true )
+      sudo( "if ! lvs /dev/#{vg}/#{lv_cache}; then", close: "fi" ) do
+        unmount_device( raw_device )
+        sudo <<-SH
+          vgextend #{vgextend_flags.join ' '} #{vg} #{raw_device}
+          lvcreate -L #{meta_size} -n #{lv_cache_meta} #{vg} #{raw_device}
+          lvcreate #{cache_size_flag} -n #{lv_cache} #{vg} #{raw_device}
+          lvconvert --type cache-pool --cachemode writethrough --yes \
+                    --poolmetadata #{vg}/#{lv_cache_meta} #{vg}/#{lv_cache}
+          lvconvert --type cache --cachepool #{vg}/#{lv_cache} #{vg}/#{lv_cache_target}
+        SH
+      end
+    end
+
+    protected
+
+    def cache_size_flag
+      if cache_size =~ /%/
+        "-l #{cache_size}"
+      else
+        "-L #{cache_size}"
+      end
+    end
+  end
+
+end

data/lib/syncwrap/components/mdraid.rb

@@ -144,15 +144,6 @@ module SyncWrap
 
     private
 
-    def unmount_device( dev )
-      sudo <<-SH
-        if mount | grep -q '^#{dev} '; then
-          umount #{dev}
-          sed -r -i '\\|^#{dev}\\s|d' /etc/fstab
-        fi
-      SH
-    end
-
     def create_raid( md )
       rlevel = raid_level || default_raid_level
       sudo <<-SH

data/lib/syncwrap/components/postgresql.rb

@@ -195,6 +195,14 @@ module SyncWrap
     # (Default: 300MB if #pg_version < 9.3)
     attr_writer :shared_memory_max
 
+    # A command pattern to initialize the database on first
+    # install. This is used on systemd distro's only.  The pattern is
+    # expanded using #pg_data_dir as the first (optional)
+    # replacement. The command is run by the postgres user.
+    #
+    # (Default: "/usr/bin/initdb %s")
+    attr_accessor :initdb_cmd
+
     def shared_memory_max
       @shared_memory_max || ( version_lt?(pg_version, [9,3]) && 300_000_000 )
     end
@@ -232,6 +240,7 @@ module SyncWrap
       @network_v4_mask = nil
       @network_v6_mask = nil
       @shared_memory_max = nil
+      @initdb_cmd = "/usr/bin/initdb %s"
       super
     end
 
@@ -281,7 +290,7 @@ module SyncWrap
             chown postgres:postgres #{pg_data_dir}
             chmod 700 #{pg_data_dir}
           SH
-          dist_service( service_name, 'initdb' )
+          pg_initdb
         end
 
       when Debian
@@ -300,6 +309,20 @@ module SyncWrap
       changes
     end
 
+    def pg_initdb
+      if distro.systemd?
+        if initdb_cmd
+          sudo <<-SH
+            su postgres -c '#{initdb_cmd % [ pg_data_dir ]}'
+          SH
+        else
+          raise ContextError, "PostgreSQL#initdb_cmd is required with systemd"
+        end
+      else
+        dist_service( service_name, 'initdb' )
+      end
+    end
+
     # Update the \PostgreSQL configuration files
     def pg_configure
       files  = %w[ pg_hba.conf pg_ident.conf postgresql.conf ]

data/lib/syncwrap/components/puma.rb

@@ -49,7 +49,7 @@ module SyncWrap
         control: "unix://#{rack_path}/control",
         environment: "production",
         port: 5874,
-        daemon: true }.merge( @puma_flags )
+        daemon: !foreground? }.merge( @puma_flags )
     end
 
     protected
@@ -66,6 +66,11 @@ module SyncWrap
       @always_restart
     end
 
+    # The name of the systemd unit file to create for this instance of
+    # puma. If specified, the name should include a '.service' suffix.
+    # (Default: nil -> no unit)
+    attr_accessor :systemd_unit
+
     public
 
     def initialize( opts = {} )
@@ -74,6 +79,7 @@ module SyncWrap
       @change_key = nil
       @rack_path = nil
       @puma_flags = {}
+      @systemd_unit = nil
       super
     end
 
@@ -83,12 +89,28 @@ module SyncWrap
       end
 
       changes = change_key && state[ change_key ]
-      rudo( "( cd #{rack_path}", close: ')' ) do
-        rudo( "if [ -f puma.state -a -e control ]; then", close: else_start ) do
-          if ( change_key && !changes ) && !always_restart?
-            rudo 'true' #no-op
-          else
-            restart
+
+      if systemd_unit
+        uchanges = rput( '/etc/systemd/system/puma.service',
+                         "/etc/systemd/system/#{systemd_unit}",
+                         user: :root )
+        if !uchanges.empty?
+          systemctl( 'enable', systemd_unit )
+        end
+        if( change_key.nil? || changes || uchanges || always_restart? )
+          systemctl( 'restart', systemd_unit )
+        else
+          systemctl( 'start', systemd_unit )
+        end
+      else
+        rudo( "( cd #{rack_path}", close: ')' ) do
+          rudo( "if [ -f puma.state -a -e control ]; then",
+                close: bare_else_start ) do
+            if ( change_key && !changes ) && !always_restart?
+              rudo 'true' #no-op
+            else
+              bare_restart
+            end
           end
         end
       end
@@ -97,11 +119,16 @@ module SyncWrap
 
     protected
 
-    def restart
+    # By default, runs in foreground if a systemd_unit is specified.
+    def foreground?
+      !!systemd_unit
+    end
+
+    def bare_restart
       rudo( ( pumactl_command + %w[ --state puma.state restart ] ).join( ' ' ) )
     end
 
-    def else_start
+    def bare_else_start
       <<-SH
         else
           #{puma_start_command}
@@ -113,7 +140,7 @@ module SyncWrap
       args = puma_flags.map do |key,value|
         if value.is_a?( TrueClass )
           key_to_arg( key )
-        elsif value.is_a?( FalseClass )
+        elsif value.nil? || value.is_a?( FalseClass )
           nil
         else
           [ key_to_arg( key ), value && value.to_s ]
@@ -126,7 +153,8 @@ module SyncWrap
       if puma_version
         [ ruby_command, '-S', 'puma', "_#{puma_version}_" ]
       else
-        [ "#{bundle_install_bin_stubs}/puma" ]
+        [ File.expand_path( File.join( bundle_install_bin_stubs, "puma" ),
+                            rack_path ) ]
       end
     end
 
@@ -134,7 +162,8 @@ module SyncWrap
       if puma_version
         [ ruby_command, '-S', 'pumactl', "_#{puma_version}_" ]
       else
-        [ "#{bundle_install_bin_stubs}/pumactl" ]
+        [ File.expand_path( File.join( bundle_install_bin_stubs, "pumactl" ),
+                            rack_path ) ]
       end
     end
 

data/lib/syncwrap/components/rhel.rb

@@ -15,14 +15,18 @@
 #++
 
 require 'syncwrap/component'
+require 'syncwrap/version_support'
 require 'syncwrap/distro'
+require 'syncwrap/systemd'
 
 module SyncWrap
 
   # Customizations for RedHat Enterprise Linux and base class for
   # derivatives like CentOS and AmazonLinux.
   class RHEL < Component
-    include SyncWrap::Distro
+    include Distro
+    include VersionSupport
+    include SystemD
 
     # RHEL version, i.e. '6'. No default value.
     attr_accessor :rhel_version
@@ -33,6 +37,10 @@ module SyncWrap
       super
     end
 
+    def systemd?
+      @is_systemd ||= version_gte?( rhel_version, [7] )
+    end
+
     # Install the specified package names. A trailing hash is
     # interpreted as options, see below.
     #
@@ -76,15 +84,25 @@ module SyncWrap
       sudo "/sbin/chkconfig --add #{name}"
     end
 
-    # Enable the System V style init.d service
+    # Enable a service by (short) name either via RHEL/System V
+    # `chkconfig on` or systemd `systemctl enable`.
     def dist_enable_init_service( name )
-      sudo "/sbin/chkconfig #{name} on"
+      if systemd?
+        systemctl( 'enable', dot_service( name ) )
+      else
+        sudo "/sbin/chkconfig #{name} on"
+      end
     end
 
-    # Run via sudo, the service command typically supporting 'start',
-    # 'stop', 'restart', 'status', etc. arguments.
+    # Run the service command typically supporting 'start', 'stop',
+    # 'restart', 'status', etc. actions.
+    # Arguments are in order: shortname, action
     def dist_service( *args )
-      sudo( [ '/sbin/service', *args ].join( ' ' ) )
+      if systemd?
+        dist_service_via_systemctl( *args )
+      else
+        sudo( [ '/sbin/service', *args ].join( ' ' ) )
+      end
     end
 
   end

data/lib/syncwrap/components/ubuntu.rb

@@ -32,6 +32,7 @@ module SyncWrap
     # an approximation of the LTS lineage.
     def debian_version
       super ||
+        ( version_gte?( ubuntu_version, [15,4] ) && '8' ) ||
         ( version_gte?( ubuntu_version, [14,4] ) && '7' ) ||
         ( version_gte?( ubuntu_version, [12,4] ) && '6' )
     end

data/lib/syncwrap/distro.rb

@@ -55,18 +55,46 @@ module SyncWrap
     end
 
     # Install a System V style init.d script (already placed at remote
-    # /etc/init.d/<name>) via distro-specific command
+    # /etc/init.d/<name>) via distro-specific command. Note that this
+    # should not be called and may fail on a distro running systemd. A
+    # rough equivalent in this case is:
+    #
+    #   systemctl( 'enable', 'name.service' )
+    #
+    # See #systemd?, SystemD#systemctl
+    #
     def dist_install_init_service( name )
       raise "Include a distro-specific component, e.g. Debian, RHEL"
     end
 
-    # Run via sudo, the System V style, distro specific `service`
-    # command, typically supporting 'start', 'stop', 'restart',
-    # 'status', etc. arguments.
+    # Run via sudo as root, either a System V (distro specific)
+    # `service` command or the systemd `systemctl` equivalent.  In the
+    # System V case arguments are passed verbatim and are in the form:
+    # (name, command, options...) Typically supported commands
+    # include: start, stop, restart, reload and status.  For maximum
+    # compatibility, in the systemd case only two arguments are
+    # allowed: shortname, command. Note the order is reversed from the
+    # use of systemctl.
     def dist_service( *args )
       raise "Include a distro-specific component, e.g. Debian, RHEL"
     end
 
+    # If found mounted, unmount the specified device and also remove
+    # it from fstab.
+    def unmount_device( dev )
+      sudo <<-SH
+        if mount | grep -q '^#{dev} '; then
+          umount #{dev}
+          sed -r -i '\\|^#{dev}\\s|d' /etc/fstab
+        fi
+      SH
+    end
+
+    # Is the Distro running systemd?
+    def systemd?
+      false
+    end
+
   end
 
 end

data/lib/syncwrap/hash_support.rb

@@ -0,0 +1,64 @@
+#--
+# Copyright (c) 2011-2015 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+module SyncWrap
+
+  # Support module for using a crytographic hash to verify the
+  # integrity of a file, for example a downloaded package.
+  module HashSupport
+
+    protected
+
+    # Enqueue a bash command to verify the specified file using the
+    # given hash. This command will fail if the file does not
+    # match. The hash :method can be specified in opts (i.e. :sha256)
+    # or will be guessed from the provided hex-encoded hash string
+    # length. Other opts are as per Component#sh, though options
+    # accept, pipefail and error are ignored. Use of :user can allow
+    # this command to be merged.
+    def hash_verify( hash, file, opts = {} )
+      opts = opts.dup
+      mth = ( opts.delete(:method) || guess_hash_method( hash ) ) or
+        raise( "Hash method unspecified and hash length " +
+               "#{hash.length} is non-standard" )
+
+      [ :accept, :pipefail, :error ].each { |k| opts.delete( k ) }
+
+      sh( <<-SH, opts )
+        echo "#{hash}  #{file}" | /usr/bin/#{mth}sum -c -
+      SH
+    end
+
+    def guess_hash_method( hash )
+      case hash.length
+      when 32
+        :md5
+      when 40
+        :sha1
+      when 56
+        :sha224
+      when 64
+        :sha256
+      when 96
+        :sha384
+      when 128
+        :sha512
+      end
+    end
+
+  end
+
+end

data/lib/syncwrap/host.rb

@@ -104,6 +104,14 @@ module SyncWrap
       end
     end
 
+    # Returns components under the specified role
+    def components_in_roles( roles )
+      roles.inject([]) do |m,r|
+        m += space.role( r ) if roles.include?( r )
+        m
+      end
+    end
+
     # Return the last component added to this Host prior to the given
     # component (either directly or via a role), or nil if there is no
     # such component.

data/lib/syncwrap/main.rb

@@ -40,6 +40,11 @@ module SyncWrap
       space.host( *args )
     end
 
+    # Shorthand for space.compose
+    def compose( &block )
+      space.compose(&block)
+    end
+
     # Merge options given, or (without opts) return space.default_options
     def options( opts = nil ) # :doc:
       if opts

data/lib/syncwrap/systemd.rb

@@ -0,0 +1,43 @@
+#--
+# Copyright (c) 2011-2015 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+module SyncWrap
+
+  # Support module for the systemd service manager, PID 1
+  module SystemD
+
+    # Run systemd `systemctl` command with args via sudo as root.
+    def systemctl( *args )
+      sudo "/usr/bin/systemctl #{args.join ' '}"
+    end
+
+    # Expand given shortname to "shortname.service" as used for the
+    # systemd unit.
+    def dot_service( shortname )
+      shortname + ".service"
+    end
+
+    protected
+
+    # Provides Distro#dist_service compatibility via #systemctl. The
+    # argument order is swapped and shortname is passed through
+    # #dot_service.
+    def dist_service_via_systemctl( shortname, action )
+      systemctl( action, dot_service( shortname ) )
+    end
+
+  end
+end

data/lib/syncwrap/user_data.rb

@@ -0,0 +1,40 @@
+#--
+# Copyright (c) 2011-2015 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+module SyncWrap
+
+  # Utility methods for generating scripts to pass as user data.
+  module UserData
+
+    private
+
+    # Returns an sh script to allow no password, no tty sudo for a
+    # specified user by writing a file to /etc/sudoers.d/<user>
+    def no_tty_sudoer( user )
+      script = <<-SH
+        #!/bin/sh -e
+        echo '#{user} ALL=(ALL) NOPASSWD:ALL'  > /etc/sudoers.d/#{user}
+        echo 'Defaults:#{user} !requiretty'   >> /etc/sudoers.d/#{user}
+        chmod 440 /etc/sudoers.d/#{user}
+      SH
+      script.split( "\n" ).map( &:strip ).join( "\n" )
+    end
+
+    module_function :no_tty_sudoer
+
+  end
+
+end

data/lib/syncwrap.rb

@@ -77,6 +77,7 @@ module SyncWrap
         sh_verbose: :v,
         sync_paths: [ File.join( SyncWrap::GEM_ROOT, 'sync' ) ] }
       @formatter = Formatter.new
+      @composed = 0
     end
 
     # Load the specified file path as per a sync.rb, into this
@@ -194,6 +195,18 @@ module SyncWrap
         uniq
     end
 
+    # Creates a new component class with &block as implementation of
+    # its :install method. Convenient for quick one-off glue but will
+    # you move to a real component class once it gets complex?
+    def compose( &block )
+      cc = Class.new(Component) do
+        define_method( :install, &block)
+      end
+      @composed += 1
+      self.class.const_set( "Composed#{@composed}", cc )
+      cc.new
+    end
+
     # Returns a new component_plan from plan, looking up any Class
     # string names and using :install for any missing methods:
     #
@@ -279,8 +292,14 @@ module SyncWrap
     private
 
     def execute_host( host, component_plan = [], opts = {} )
+      comp_roles = opts[ :comp_roles ]
+      comps = if comp_roles && !comp_roles.empty?
+                host.components_in_roles( comp_roles )
+              else
+                host.components
+              end
       # Important: resolve outside of context
-      comp_methods = resolve_component_methods(host.components, component_plan)
+      comp_methods = resolve_component_methods(comps, component_plan)
       ctx = Context.new( host, opts )
       ctx.with do
         comp_methods.each do |comp, mths|
@@ -372,6 +391,7 @@ module SyncWrap
   autoload :Iyyov,         'syncwrap/components/iyyov'
   autoload :IyyovDaemon,   'syncwrap/components/iyyov_daemon'
   autoload :JRubyVM,       'syncwrap/components/jruby_vm'
+  autoload :LVMCache,      'syncwrap/components/lvm_cache'
   autoload :MDRaid,        'syncwrap/components/mdraid'
   autoload :Network,       'syncwrap/components/network'
   autoload :OpenJDK,       'syncwrap/components/open_jdk'
@@ -392,5 +412,6 @@ module SyncWrap
   # Additional autoloads (optional support)
   autoload :AmazonEC2,     'syncwrap/amazon_ec2'
   autoload :GitHelp,       'syncwrap/git_help'
+  autoload :UserData,      'syncwrap/user_data'
 
 end

data/sync/etc/systemd/system/iyyov.service.erb

@@ -0,0 +1,16 @@
+[Unit]
+Description=Iyyov JRuby Job Control Daemon
+# Introduced in systemd v218+:
+# AssertFileIsExecutable=<%= jruby_gem_home %>/gems/iyyov-<%= iyyov_version %>-java/init/iyyov
+
+[Service]
+User=<%= run_user %>
+Type=forking
+ExecStart=<%= jruby_gem_home %>/gems/iyyov-<%= iyyov_version %>-java/init/iyyov jobs.rb
+WorkingDirectory=<%= run_dir %>/iyyov
+PIDFile=<%= run_dir %>/iyyov/iyyov.pid
+Restart=always
+ExecReload=/usr/bin/touch <%= run_dir %>/iyyov/jobs.rb
+
+[Install]
+WantedBy=multi-user.target

data/sync/etc/systemd/system/puma.service.erb

@@ -0,0 +1,13 @@
+[Unit]
+Description=Puma HTTP Server
+After=network.target
+
+[Service]
+Type=simple
+User=<%= run_user %>
+WorkingDirectory=<%= rack_path %>
+ExecStart=<%= puma_start_command %>
+Restart=always
+
+[Install]
+WantedBy=multi-user.target