syncwrap 1.3.0 → 1.4.0

data/History.rdoc

@@ -1,3 +1,11 @@
+=== 1.4.0 (2012-10-16)
+* PostgreSQL configuration updates for 9.2.x (RHEL only)
+* Add support for build, deploy, configure of Apache Qpid for which
+  stock RPMs are not available on Amazon Linux. This includes corosync
+  build for a working cluster setup on EC2.
+* Add :succeed option to RHEL.dist_install (for local RPM files that
+  may already be installed)
+
 === 1.3.0 (2012-10-4)
 * Include RHEL/Amazon Linux Postgresql 9.1 config and deploy support
   and reorganize for Ubuntu support as well. Default deploy data dir

data/Manifest.txt

@@ -3,7 +3,10 @@ Manifest.txt
 README.rdoc
 Rakefile
 etc/gemrc
+etc/corosync/corosync.conf
+etc/corosync/uidgid.d/qpid
 etc/init.d/iyyov
+etc/init.d/qpidd
 etc/sysconfig/pgsql/postgresql
 etc/sysctl.d/61-postgresql-shm.conf
 lib/syncwrap/base.rb
@@ -17,6 +20,7 @@ lib/syncwrap/iyyov.rb
 lib/syncwrap/java.rb
 lib/syncwrap/jruby.rb
 lib/syncwrap/postgresql.rb
+lib/syncwrap/qpid.rb
 lib/syncwrap/remote_task.rb
 lib/syncwrap/rhel.rb
 lib/syncwrap/ubuntu.rb
@@ -32,4 +36,5 @@ postgresql/ubuntu/postgresql.conf
 postgresql/ubuntu/start.conf
 test/test_syncwrap.rb
 usr/local/bin/jgem
+usr/local/etc/qpidd.conf
 var/iyyov/jobs.rb

data/etc/corosync/corosync.conf

@@ -0,0 +1,39 @@
+# corosync sample config using pre-determined member list and updu
+# which is EC2 compatible.  Enable UDP access to udp ports 5404 and
+# 5405 on each cluster member.
+#
+# See also http://www.thatsgeeky.com/2011/12/installing-corosync-on-ec2/
+
+compatibility: whitetank
+
+totem {
+    version: 2
+    secauth: off
+    interface {
+        member {
+            memberaddr: 10.252.133.51 #FIXME: Replace with your actual members
+        }
+        member {
+            memberaddr: 10.252.73.186
+        }
+        ringnumber: 0
+        bindnetaddr: 10.252.0.0 # FIXME: is host specific, compute via:
+        # ipcalc -n `ip addr show eth0 | grep 'inet ' |awk '{print $2}'` | awk -F= '{print $2}'
+        mcastport: 5405
+        ttl: 1
+    }
+    transport: udpu
+}
+
+logging {
+    fileline: off
+    to_logfile: yes
+    to_syslog: yes
+    logfile: /var/log/cluster/corosync.log
+    debug: off
+    timestamp: on
+    logger_subsys {
+        subsys: AMF
+        debug: off
+    }
+}

data/etc/corosync/uidgid.d/qpid

@@ -0,0 +1,4 @@
+uidgid {
+  uid: qpidd
+  gid: qpidd
+}

data/etc/init.d/qpidd

@@ -0,0 +1,89 @@
+#!/bin/bash
+#
+# qpidd        Startup script for the Qpid messaging daemon.
+#
+# chkconfig: 2345 85 15
+# description: Qpidd is an AMQP broker.
+# processname: qpidd
+# config: /usr/local/etc/qpidd.conf
+
+prog=qpidd
+lockfile=/var/lock/subsys/$prog
+pidfile=/var/run/qpidd.pid
+
+# Source function library.
+. /etc/rc.d/init.d/functions
+
+if [ -f /etc/sysconfig/$prog ] ; then
+    . /etc/sysconfig/$prog
+fi
+
+RETVAL=0
+
+#ensure binary is present and executable
+if [[ !(-x /usr/local/sbin/$prog) ]] ; then
+    echo "/usr/local/sbin/$prog not found or not executable"
+    exit 5
+fi
+
+#ensure user has sufficient permissions
+runuser -s /bin/sh qpidd -c "echo x > /dev/null" 2> /dev/null || RETVAL=4
+if [ $RETVAL = 4 ]; then
+    echo "user had insufficient privilege";
+    exit $RETVAL
+fi
+
+start() {
+    echo -n $"Starting Qpid AMQP daemon: "
+    daemon --pidfile $pidfile --check $prog --user qpidd /usr/local/sbin/$prog --daemon $QPID_OPTIONS
+    RETVAL=$?
+    echo
+    [ $RETVAL = 0 ] && touch $lockfile
+    if [ $RETVAL = 0 ]; then
+        touch $pidfile
+        chown qpidd.qpidd $pidfile
+        [ -x /sbin/restorecon ] && /sbin/restorecon $pidfile
+        runuser -s /bin/sh qpidd -c "/usr/local/sbin/$prog --check > $pidfile"
+    fi
+    return $RETVAL
+}
+
+stop() {
+    echo -n $"Stopping Qpid AMQP daemon: "
+    killproc -p ${pidfile} $prog
+    RETVAL=$?
+    echo
+    [ $RETVAL = 0 ] && rm -f ${lockfile} ${pidfile}
+}
+
+reload() {
+    echo 1>&2 $"$0: reload not supported"
+    exit 3
+}
+
+restart() {
+    stop
+    start
+}
+
+# See how we were called.
+case "$1" in
+  start|stop|restart|reload)
+    $1
+    ;;
+  status)
+    status $prog
+    RETVAL=$?
+    ;;
+  force-reload)
+    restart
+    ;;
+  try-restart|condrestart)
+    [ -e $lockfile ] && restart || :
+    ;;
+  *)
+    echo 1>&2 $"Usage: $0 {start|stop|status|restart|condrestart|try-restart|force-reload}"
+    exit 2
+esac
+
+exit $RETVAL

data/lib/syncwrap/base.rb

@@ -15,5 +15,5 @@
 #++
 
 module SyncWrap
-  VERSION='1.3.0'
+  VERSION='1.4.0'
 end

data/lib/syncwrap/postgresql.rb

@@ -50,6 +50,7 @@ module SyncWrap::PostgreSQL
   # Update PostgreSQL config files
   def pg_configure
     rput( "#{pg_deploy_config}/", pg_config_dir, :user => 'postgres' )
+    sudo( "chmod 700 #{pg_data_dir}" ) if pg_config_dir == pg_data_dir
   end
 
   def pg_start

data/lib/syncwrap/qpid.rb

@@ -0,0 +1,251 @@
+#--
+# Copyright (c) 2011-2012 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You may
+# obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+require 'syncwrap/common'
+require 'syncwrap/distro'
+
+# Qpid AMQP broker provisioning. Currently this is RHEL/AmazonLinux
+# centric.
+module SyncWrap::Qpid
+  include SyncWrap::Common
+  include SyncWrap::Distro
+
+  attr_accessor :qpid_src_root
+
+  attr_accessor :qpid_version
+  attr_accessor :qpid_repo
+  attr_accessor :qpid_distro
+
+  attr_accessor :corosync_version
+  attr_accessor :corosync_repo
+
+  def initialize
+    super
+
+    @qpid_src_root = '/tmp/src'
+
+    @qpid_version = '0.18'
+    @qpid_repo = 'http://apache.osuosl.org/qpid'
+    @qpid_distro = 'amzn1'
+
+    @corosync_version = '1.4.4'
+    @corosync_repo = 'https://github.com/downloads/corosync/corosync'
+  end
+
+  def qpid_install
+    unless exist?( "/usr/local/sbin/qpidd" )
+      qpid_install!
+      qpid_install_init!
+    end
+    qpid_tools_install
+    rput( 'usr/local/etc/qpidd.conf', :user => 'root' )
+  end
+
+  def qpid_install_init!
+    unless exec_conditional { run "id qpidd >/dev/null" } == 0
+      sudo <<-SH
+        useradd -r qpidd
+        mkdir -p /var/local/qpidd
+        chown qpidd:qpidd /var/local/qpidd
+      SH
+    end
+
+    rput( 'etc/init.d/qpidd', :user => 'root' )
+
+    # Add to init.d
+    dist_install_init_service( 'qpidd' )
+  end
+
+  def qpid_tools_install
+    unless exist?( "/usr/bin/qpid-config" )
+      qpid_tools_install!
+    end
+  end
+
+  def qpid_install!
+    corosync_install!( :devel => true )
+    qpid_build
+  end
+
+  def qpid_tools_install!
+    dist_install( 'python-setuptools' )
+    qpid_src = "#{qpid_src_root}/qpid-#{qpid_version}"
+
+    run <<-SH
+      mkdir -p #{qpid_src_root}
+      rm -rf #{qpid_src}
+      cd #{qpid_src_root}
+      curl -sSL #{qpid_tools_tarball} | tar -zxf -
+    SH
+
+    sudo <<-SH
+      cd #{qpid_src}
+      easy_install ./python ./tools ./extras/qmf
+    SH
+  end
+
+  def qpid_build
+    qpid_install_build_deps
+    qpid_install_deps
+
+    qpid_src = "#{qpid_src_root}/qpidc-#{qpid_version}"
+
+    run <<-SH
+      mkdir -p #{qpid_src_root}
+      rm -rf #{qpid_src}
+      cd #{qpid_src_root}
+      curl -sSL #{qpid_src_tarball} | tar -zxf -
+      cd #{qpid_src}
+      ./configure > /dev/null
+      make > /dev/null
+    SH
+
+    sudo <<-SH
+      cd #{qpid_src}
+      make install > /dev/null
+    SH
+
+    run <<-SH
+      cd #{qpid_src}
+      make check > /dev/null
+    SH
+
+    sudo <<-SH
+      cd /usr/local
+      rm -f /tmp/qpidc-#{qpid_version}-1-#{qpid_distro}-x64.tar.gz
+      tar -zc \
+           --exclude games --exclude lib64/perl5 --exclude src \
+           --exclude share/man --exclude share/perl5 --exclude share/info \
+           --exclude share/applications \
+           -f /tmp/qpidc-#{qpid_version}-1-#{qpid_distro}-x64.tar.gz .
+    SH
+  end
+
+  def qpid_src_tarball
+    "#{qpid_repo}/#{qpid_version}/qpid-cpp-#{qpid_version}.tar.gz"
+  end
+
+  def qpid_tools_tarball
+    "#{qpid_repo}/#{qpid_version}/qpid-#{qpid_version}.tar.gz"
+  end
+
+  def qpid_install_build_deps
+    dist_install( %w[ gcc gcc-c++ make autogen autoconf
+                      help2man libtool pkgconfig rpm-build ] )
+  end
+
+  def qpid_install_deps
+    dist_install( %w[ nss-devel boost-devel libuuid-devel swig
+                      ruby-devel python-devel cyrus-sasl-devel ] )
+  end
+
+  def corosync_install( opts = {} )
+    unless exist?( "/usr/sbin/corosync" )
+      corosync_install!( opts )
+    end
+  end
+
+  def corosync_install!( opts = {} )
+    corosync_build
+    dist_install( "#{corosync_src}/x86_64/*.rpm", :succeed => true )
+  end
+
+  def corosync_build
+    qpid_install_build_deps
+    corosync_install_build_deps
+
+    run <<-SH
+      mkdir -p #{qpid_src_root}
+      rm -rf #{corosync_src}
+      cd #{qpid_src_root}
+      curl -sSL #{corosync_repo}/corosync-#{corosync_version}.tar.gz | tar -zxf -
+      cd #{corosync_src}
+      ./autogen.sh
+      ./configure > /dev/null
+      make rpm > /dev/null
+    SH
+
+  end
+
+  def corosync_packages( include_devel = false )
+    packs = [ "corosync-#{corosync_version}-1.#{qpid_distro}.x86_64.rpm",
+              "corosynclib-#{corosync_version}-1.#{qpid_distro}.x86_64.rpm" ]
+    packs <<  "corosynclib-devel-#{corosync_version}-1.#{qpid_distro}.x86_64.rpm" if include_devel
+    packs
+  end
+
+  def corosync_install_build_deps
+    dist_install( %w[ nss-devel libibverbs-devel librdmacm-devel ] )
+  end
+
+  def corosync_src
+    "#{qpid_src_root}/corosync-#{corosync_version}"
+  end
+
+  # Simplify qpid install by using pre-built binaries (for example,
+  # archived from the build steps above.)
+  module BinRepo
+    include SyncWrap::Qpid
+
+    attr_accessor :qpid_prebuild_repo
+
+    def initialize
+      super
+
+      @qpid_prebuild_repo = nil
+    end
+
+    def qpid_install
+      corosync_install
+      super
+    end
+
+    def qpid_install!
+      raise "qpid_prebuild_repo required, but not set" unless qpid_prebuild_repo
+
+      dist_install( %w[ boost cyrus-sasl ] )
+
+      sudo <<-SH
+        cd /usr/local
+        curl -sS #{qpid_prebuild_repo}/qpidc-#{qpid_version}-1-#{qpid_distro}-x64.tar.gz | tar -zxf -
+      SH
+    end
+
+    def corosync_install!( opts = {} )
+      raise "qpid_prebuild_repo required, but not set" unless qpid_prebuild_repo
+      packs = corosync_packages
+      curls = packs.map do |p|
+        "curl -sS -O #{qpid_prebuild_repo}/#{p}"
+      end
+
+      sudo <<-SH
+        rm -rf /tmp/rpm-drop
+        mkdir -p /tmp/rpm-drop
+        cd /tmp/rpm-drop
+        #{curls.join("\n")}
+      SH
+      dist_install( "/tmp/rpm-drop/*.rpm", :succeed => true )
+    end
+
+    # Where uploaded qpid-python-tools-M.N.tar.gz contains the
+    # ./python ./tools ./extras/qmf packages for easy_install.
+    def qpid_tools_tarball
+      "#{qpid_prebuild_repo}/qpid-python-tools-#{qpid_version}.tar.gz"
+    end
+
+  end
+
+end

data/lib/syncwrap/rhel.rb

@@ -29,9 +29,27 @@ module SyncWrap::RHEL
                          'postgresql' => 'postgresql-server' )
   end
 
+  # Install packages. The last argument is interpreted as a options if
+  # it is a Hash.
+  # === Options
+  # :succeed:: Always succeed (useful for local rpms which might
+  # already be installed.
   def dist_install( *pkgs )
+
+    if pkgs.last.is_a?( Hash )
+      pkgs = pkgs.dup
+      opts = pkgs.pop
+    else
+      opts = {}
+    end
+
     pkgs = dist_map_packages( pkgs )
-    sudo "yum install -q -y #{pkgs.join( ' ' )}"
+
+    if opts[ :succeed ]
+      sudo "yum install -q -y #{pkgs.join( ' ' )} || true"
+    else
+      sudo "yum install -q -y #{pkgs.join( ' ' )}"
+    end
   end
 
   def dist_uninstall( *pkgs )

data/postgresql/rhel/postgresql.conf

@@ -45,7 +45,7 @@
                                         # (change requires restart)
 
 # If external_pid_file is not explicitly set, no extra PID file is written.
-#external_pid_file = '(none)'           # write an extra PID file
+#external_pid_file = ''                 # write an extra PID file
                                         # (change requires restart)
 
 #------------------------------------------------------------------------------
@@ -56,14 +56,17 @@
 
 #listen_addresses = 'localhost'         # what IP address(es) to listen on;
                                         # comma-separated list of addresses;
-                                        # defaults to 'localhost', '*' = all
+                                        # defaults to 'localhost'; use '*' for all
                                         # (change requires restart)
 #port = 5432                            # (change requires restart)
+# Note: In RHEL/Fedora installations, you can't set the port number here;
+# adjust it in the service file instead.
 max_connections = 100                   # (change requires restart)
 # Note:  Increasing max_connections costs ~400 bytes of shared memory per
 # connection slot, plus lock space (see max_locks_per_transaction).
 #superuser_reserved_connections = 3     # (change requires restart)
-#unix_socket_directory = ''             # (change requires restart)
+#unix_socket_directories = '/var/run/postgresql, /tmp'  # comma-separated list of directories
+                                        # (change requires restart)
 #unix_socket_group = ''                 # (change requires restart)
 #unix_socket_permissions = 0777         # begin with 0 to use octal notation
                                         # (change requires restart)
@@ -79,6 +82,10 @@ max_connections = 100                   # (change requires restart)
 #ssl_ciphers = 'ALL:!ADH:!LOW:!EXP:!MD5:@STRENGTH'      # allowed SSL ciphers
                                         # (change requires restart)
 #ssl_renegotiation_limit = 512MB        # amount of data between renegotiations
+#ssl_cert_file = 'server.crt'           # (change requires restart)
+#ssl_key_file = 'server.key'            # (change requires restart)
+#ssl_ca_file = ''                       # (change requires restart)
+#ssl_crl_file = ''                      # (change requires restart)
 #password_encryption = on
 #db_user_namespace = off
 
@@ -177,6 +184,9 @@ checkpoint_segments = 16                # in logfile segments, min 1, 16MB each
 #archive_mode = off             # allows archiving to be done
                                 # (change requires restart)
 #archive_command = ''           # command to use to archive a logfile segment
+                                # placeholders: %p = path of file to archive
+                                #               %f = file name only
+                                # e.g. 'test ! -f /mnt/server/archivedir/%f && cp %p /mnt/server/archivedir/%f'
 #archive_timeout = 0            # force a logfile segment switch after this
                                 # number of seconds; 0 disables
 
@@ -186,21 +196,25 @@ checkpoint_segments = 16                # in logfile segments, min 1, 16MB each
 
 # - Master Server -
 
-# These settings are ignored on a standby server
+# Set these on the master and on any standby that will send replication data.
 
 #max_wal_senders = 0            # max number of walsender processes
                                 # (change requires restart)
-#wal_sender_delay = 1s          # walsender cycle time, 1-10000 milliseconds
 #wal_keep_segments = 0          # in logfile segments, 16MB each; 0 disables
-#vacuum_defer_cleanup_age = 0   # number of xacts by which cleanup is delayed
 #replication_timeout = 60s      # in milliseconds; 0 disables
+
+# - Master Server -
+
+# These settings are ignored on a standby server.
+
 #synchronous_standby_names = '' # standby servers that provide sync rep
                                 # comma-separated list of application_name
                                 # from standby(s); '*' = all
+#vacuum_defer_cleanup_age = 0   # number of xacts by which cleanup is delayed
 
 # - Standby Servers -
 
-# These settings are ignored on a master server
+# These settings are ignored on a master server.
 
 #hot_standby = off                      # "on" allows queries during recovery
                                         # (change requires restart)
@@ -225,6 +239,7 @@ checkpoint_segments = 16                # in logfile segments, min 1, 16MB each
 #enable_hashagg = on
 #enable_hashjoin = on
 #enable_indexscan = on
+#enable_indexonlyscan = on
 #enable_material = on
 #enable_mergejoin = on
 #enable_nestloop = on
@@ -302,10 +317,8 @@ log_rotation_size = 0                   # Automatic rotation of logfiles will
 #syslog_facility = 'LOCAL0'
 #syslog_ident = 'postgres'
 
-#silent_mode = off                      # Run server silently.
-                                        # DO NOT USE without syslog or
-                                        # logging_collector
-                                        # (change requires restart)
+# This is only relevant when logging to eventlog (win32):
+#event_source = 'PostgreSQL'
 
 # - When to Log -
 
@@ -390,7 +403,7 @@ log_line_prefix = '%t '                 # special values:
 #log_temp_files = -1                    # log temporary files equal or larger
                                         # than the specified size in kilobytes;
                                         # -1 disables, 0 logs all temp files
-#log_timezone = '(defaults to server environment setting)'
+log_timezone = 'UTC'
 
 #------------------------------------------------------------------------------
 # RUNTIME STATISTICS
@@ -400,6 +413,7 @@ log_line_prefix = '%t '                 # special values:
 
 #track_activities = on
 #track_counts = on
+#track_io_timing = off
 #track_functions = none                 # none, pl, all
 #track_activity_query_size = 1024       # (change requires restart)
 #update_process_title = on
@@ -466,7 +480,7 @@ log_line_prefix = '%t '                 # special values:
 
 datestyle = 'iso, mdy'
 #intervalstyle = 'postgres'
-#timezone = '(defaults to server environment setting)'
+timezone = 'UTC'
 #timezone_abbreviations = 'Default'     # Select the set of available time zone
                                         # abbreviations.  Currently, there are
                                         #   Default
@@ -530,11 +544,11 @@ default_text_search_config = 'pg_catalog.english'
 # ERROR HANDLING
 #------------------------------------------------------------------------------
 
-#exit_on_error = off                            # terminate session on any error?
-#restart_after_crash = on                       # reinitialize after backend crash?
+#exit_on_error = off                    # terminate session on any error?
+#restart_after_crash = on               # reinitialize after backend crash?
 
 #------------------------------------------------------------------------------
 # CUSTOMIZED OPTIONS
 #------------------------------------------------------------------------------
 
-#custom_variable_classes = ''           # list of custom variable class names
+# Add settings for extensions here

data/usr/local/etc/qpidd.conf

@@ -0,0 +1,11 @@
+auth=no
+
+data-dir=/var/local/qpidd
+pid-dir=/var/local/qpidd
+# log-to-file=/var/local/qpidd/qpidc.log
+
+# For clustering, ensure corosync is configured and uncomment the
+# following.
+
+# cluster-mechanism=ANONYMOUS
+# cluster-name=qpid

metadata

@@ -2,14 +2,14 @@
 name: syncwrap
 version: !ruby/object:Gem::Version
   prerelease:
-  version: 1.3.0
+  version: 1.4.0
 platform: ruby
 authors:
 - David Kellum
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2012-10-04 00:00:00.000000000 Z
+date: 2012-10-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -57,7 +57,10 @@ files:
 - README.rdoc
 - Rakefile
 - etc/gemrc
+- etc/corosync/corosync.conf
+- etc/corosync/uidgid.d/qpid
 - etc/init.d/iyyov
+- etc/init.d/qpidd
 - etc/sysconfig/pgsql/postgresql
 - etc/sysctl.d/61-postgresql-shm.conf
 - lib/syncwrap/base.rb
@@ -71,6 +74,7 @@ files:
 - lib/syncwrap/java.rb
 - lib/syncwrap/jruby.rb
 - lib/syncwrap/postgresql.rb
+- lib/syncwrap/qpid.rb
 - lib/syncwrap/remote_task.rb
 - lib/syncwrap/rhel.rb
 - lib/syncwrap/ubuntu.rb
@@ -86,6 +90,7 @@ files:
 - postgresql/ubuntu/start.conf
 - test/test_syncwrap.rb
 - usr/local/bin/jgem
+- usr/local/etc/qpidd.conf
 - var/iyyov/jobs.rb
 homepage: http://github.com/dekellum/syncwrap
 licenses: []