syncwrap 1.5.2 → 2.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 7750e1149348fc952ef8d2dd124282861961bc4f
+  data.tar.gz: 1c183faa0793ee65741b64366e1463b3c4593e51
+SHA512:
+  metadata.gz: db6263d45a815839d43aed07c08a22cfc5dc578cbcb9c3438e3ab74ba619ba78dd3d8780beaf4648d27336284192cd4112b9e2ce6edd7d3029b5089d6db441fa
+  data.tar.gz: 694c6a0e63d9358f6831f21f16bfe9f2caffcf51c27080689e85545e24a9ce1f4b0c0ec3a44fcaa5610586e92545e1b7ad11c52e4fb696405fdde281fc0c6b1f

data/History.rdoc

@@ -1,3 +1,22 @@
+=== 2.0.0 (2014-2-26)
+* Major rewrite with only very limited conceptual compatibility with
+  1.x. See README, LAYOUT, and examples.
+* Now requires ruby 1.9 stdlib. Internally makes extensive use of new
+  Hash syntax.
+* New CLI, `syncwrap` command and sync.rb file replaces
+  Vlad/RemoteTask and rake (though rake integration is still
+  possible.) SyncWrap 2 is a standalone complete solution.
+* Component classes and dynamic binding replaces the subject-oriented
+  module mixins.
+* Block oriented, thread safe, colored console output
+* SyncWrap::Component#sh replaces `run`, includes a command queue
+  and compossible scripts.
+* SyncWrap::Component#rput with multiple sync_paths and transparent
+  ERB template support.
+* Extensive updates, templating, reliability improvements, and
+  optimization of existing provisions (now Components)
+* New Iyyov jobs.d support and standardized SyncWrap::IyyovDeamon
+
 === 1.5.2 (2013-4-10)
 * Add check for init, aws_configure only if aws_config_json file
   exists.

data/Manifest.txt

@@ -2,40 +2,88 @@ History.rdoc
 Manifest.txt
 README.rdoc
 Rakefile
-etc/gemrc
-etc/corosync/corosync.conf
-etc/corosync/uidgid.d/qpid
-etc/init.d/iyyov
-etc/init.d/qpidd
-etc/sysconfig/pgsql/postgresql
-etc/sysctl.d/61-postgresql-shm.conf
+bin/syncwrap
+examples/LAYOUT.rdoc
+examples/Rakefile
+examples/ec2.rb
+examples/hello.rb
+examples/hello_binding.rb
+examples/jruby.rb
+examples/rput.rb
+examples/private/aws.json
+examples/sync/home/bob/.ssh/authorized_keys
+examples/sync/tmp/sample.erb
 lib/syncwrap/base.rb
 lib/syncwrap.rb
-lib/syncwrap/aws.rb
-lib/syncwrap/common.rb
+lib/syncwrap/amazon_ec2.rb
+lib/syncwrap/amazon_ws.rb
+lib/syncwrap/cli.rb
+lib/syncwrap/component.rb
+lib/syncwrap/context.rb
 lib/syncwrap/distro.rb
-lib/syncwrap/ec2.rb
-lib/syncwrap/geminabox.rb
-lib/syncwrap/hashdot.rb
-lib/syncwrap/iyyov.rb
-lib/syncwrap/java.rb
-lib/syncwrap/jruby.rb
-lib/syncwrap/postgresql.rb
-lib/syncwrap/qpid.rb
-lib/syncwrap/remote_task.rb
-lib/syncwrap/rhel.rb
-lib/syncwrap/ubuntu.rb
-lib/syncwrap/user_run.rb
-postgresql/rhel/pg_hba.conf
-postgresql/rhel/pg_ident.conf
-postgresql/rhel/postgresql.conf
-postgresql/ubuntu/environment
-postgresql/ubuntu/pg_ctl.conf
-postgresql/ubuntu/pg_hba.conf
-postgresql/ubuntu/pg_ident.conf
-postgresql/ubuntu/postgresql.conf
-postgresql/ubuntu/start.conf
-test/test_syncwrap.rb
-usr/local/bin/jgem
-usr/local/etc/qpidd.conf
-var/iyyov/jobs.rb
+lib/syncwrap/formatter.rb
+lib/syncwrap/host.rb
+lib/syncwrap/main.rb
+lib/syncwrap/path_util.rb
+lib/syncwrap/rsync.rb
+lib/syncwrap/ruby_support.rb
+lib/syncwrap/shell.rb
+lib/syncwrap/components/commercial_jdk.rb
+lib/syncwrap/components/cruby_vm.rb
+lib/syncwrap/components/etc_hosts.rb
+lib/syncwrap/components/geminabox.rb
+lib/syncwrap/components/hashdot.rb
+lib/syncwrap/components/iyyov.rb
+lib/syncwrap/components/iyyov_daemon.rb
+lib/syncwrap/components/jruby_vm.rb
+lib/syncwrap/components/mdraid.rb
+lib/syncwrap/components/network.rb
+lib/syncwrap/components/open_jdk.rb
+lib/syncwrap/components/postgresql.rb
+lib/syncwrap/components/qpid.rb
+lib/syncwrap/components/rhel.rb
+lib/syncwrap/components/run_user.rb
+lib/syncwrap/components/ubuntu.rb
+lib/syncwrap/components/users.rb
+sync/etc/gemrc
+sync/etc/hosts.erb
+sync/etc/corosync/corosync.conf
+sync/etc/corosync/uidgid.d/qpid
+sync/etc/init.d/iyyov.erb
+sync/etc/init.d/qpidd
+sync/etc/sysconfig/pgsql/postgresql.erb
+sync/etc/sysctl.d/61-postgresql-shm.conf
+sync/jruby/bin/jgem
+sync/postgresql/rhel/pg_hba.conf
+sync/postgresql/rhel/pg_ident.conf
+sync/postgresql/rhel/postgresql.conf
+sync/postgresql/ubuntu/environment
+sync/postgresql/ubuntu/pg_ctl.conf
+sync/postgresql/ubuntu/pg_hba.conf
+sync/postgresql/ubuntu/pg_ident.conf
+sync/postgresql/ubuntu/postgresql.conf
+sync/postgresql/ubuntu/start.conf
+sync/src/hashdot/Makefile.erb
+sync/src/hashdot/profiles/default.hdp.erb
+sync/src/hashdot/profiles/jruby-common.hdp
+sync/src/hashdot/profiles/jruby-shortlived.hdp
+sync/src/hashdot/profiles/jruby.hdp.erb
+sync/src/hashdot/profiles/shortlived.hdp
+sync/usr/local/etc/qpidd.conf
+sync/var/iyyov/jobs.rb.erb
+sync/var/iyyov/default/config.rb
+sync/var/iyyov/default/daemon.rb.erb
+test/muddled_sync.rb
+test/setup.rb
+test/test_components.rb
+test/test_context.rb
+test/test_context_rput.rb
+test/test_rsync.rb
+test/test_shell.rb
+test/test_space.rb
+test/test_space_main.rb
+test/zfile
+test/sync/d1/bar
+test/sync/d1/foo.erb
+test/sync/d3/d2/bar
+test/sync/d3/d2/foo.erb

data/README.rdoc

@@ -1,85 +1,133 @@
 = SyncWrap
 
 * http://github.com/dekellum/syncwrap
+* {<img src="https://travis-ci.org/dekellum/syncwrap.png" />}[https://travis-ci.org/dekellum/syncwrap]
 
 == Description
 
-A generalized and modular set of provisioning and deployment routines.
-The Rake-centric Vlad and not-actually-Rake Capistrano both suffer
-from lack of objects, e.g. the ability to customize and mix-in
-behavior. SyncWrap offers an Object/Module system that can be used
-with either.
+A rather direct provisioning and deployment system in ruby, bash over
+ssh, and rsync. SyncWrap is a refreshingly straightforward, DSL-free
+non-framework with no central server or repository requirements. The
+"magic", if any, lies in the elegant component model, the shell
+command queue, and a few powerful methods like rput (augmented rsync).
 
 == Features
 
-* Multi-line shell support in SyncWrap::Common#run,
-  SyncWrap::Common#sudo for more natural script embedding
+* Default Thread-per-host parallel operation, with synchronized
+  block/live console output for debugging.
 
-* SyncWrap::Common::rput offers some useful rsync magic, like setting
-  the owner of placed remote files
+* Multi-line shell support (often via here-document), command queue,
+  and composition support in SyncWrap::Component#sh for more natural
+  script embedding.  You'll know you are provisioning with bash, in
+  all its syntactic glory.
 
-* Platform differences are factored out into platform-specific modules
+* SyncWrap::Component#rput wraps rsync, with support for multiple
+  source roots, transparent ERB templates, and target file
+  owner/permissions control. Returns the file change status allowing
+  reactive scripting (i.e. restart daemon since config has changed.)
 
-* SyncWrap::RemoteTask rake-remote_task adapter (see synopsis below)
+* Linux distro differences are factored out into distro-specific
+  components with utility methods exposed in host context.
 
-* Gem install utilities. e.g. SyncWrap::JRuby#jruby_check_gem,
-  SyncWrap::JRuby#jruby_install_gem
+* SyncWrap::AmazonEC2 provider with host profiles: create and
+  provision hosts in a single step.
 
-== Provisions
+* Independent of (but easily integrated into) rake
 
-Currently the following provisions are provided:
+* Depends only on ruby 1.9+ stdlib Open3 for interaction with bash,
+  ssh and rsync. Actually, really works on jruby 1.6.8 (--1.9 mode)
+  and 1.7+
 
-* SyncWrap::AWS for EC2 instances, mdraid over EBS volumes, and
-  Route53 DNS updates.
+== Synopsis
 
-* SyncWrap::PostgreSQL
+Simplest example. Place the following in a sync.rb file:
 
-* Complete SyncWrap::Java, SyncWrap::Hashdot, SyncWrap::JRuby stack
+  class Greeter < SyncWrap::Component
+    def install
+      say_it
+    end
 
-* SyncWrap::Iyyov job scheduler and process monitor
+    def say_it
+      sh <<-SH
+        echo "Hello from #{host.name}"
+      SH
+    end
+  end
 
-* SyncWrap::UserRun for setup of a run user for deployed daemons, jobs
-  and var directories.
+  host 'localhost', Greeter.new
 
-== Synopsis
+Try it via the command line:
 
-For example, in your Rakefile with the SyncWrap::RemoteTask adapter:
+  gem install syncwrap
+  syncwrap --help
+  syncwrap -f ./sync.rb --list  # -f default; list
+  syncwrap                      # Run it
+  syncwrap -v                   # Run it with full verbose output
+  syncwrap -v Greeter.say_it    # Without Greeter#install default
 
-  class SyncWrapper
+Output from syncwrap -v (minus the terminal color):
 
-    # Include these support modules for the tasks below, or include
-    # your own.
+  == localhost #<Module:0x00000001e95678>::Greeter#install: enqueue
+  <-- sh localhost (-v coalesce live)
+  echo "Hello from localhost"
+  Hello from localhost
+  --> Exit 0 (success)
 
-    include SyncWrap::Java
-    include SyncWrap::Hashdot
-    include SyncWrap::JRuby
-    include SyncWrap::Ubuntu
+See more elaborate examples, including ec2 provider support, in the
+examples directory and LAYOUT.
 
-    include SyncWrap::RemoteTask
+== Components
 
-    def initialize
-      super
-      self.common_prefix = '/usr/local'
-    end
+Components implementing SyncWrap::Distro, to expose common utility
+methods like dist_install:
 
-    def define_tasks
-      desc "Combined Java, Hashdot, JRuby Deployment"
-      remote_task :jruby_deploy do
-        java_install
-        hashdot_install
-        jruby_install
-      end
-    end
+* SyncWrap::RHEL (also supports CentOS, Amazon Linux)
+* SyncWrap::Ubuntu
 
-    # Override any of the support methods as needed.
+Components implementing \#install:
 
-  end
+* SyncWrap::MDRaid - Software raid, lvm volumes, filesystems, mounts
+
+* SyncWrap::Network and SyncWrap::EtcHosts - hostname, static names,
+  resolver config
+
+* SyncWrap::Users - developer accounts, home directory files (incl. ssh
+  keys), sudoers access
+
+* SyncWrap::RunUser - a user for running deployed daemons, jobs, in
+  var directories.
+
+* SyncWrap::PostgreSQL
+
+* SyncWrap::OpenJDK or SyncWrap::CommercialJDK
+
+* SyncWrap::JRubyVM and/or SyncWrap::CRubyVM (MRI, from source)
+
+* SyncWrap::Iyyov - job scheduler and process monitor with
+  utilities for SyncWrap::IyyovDaemon deployment.
+
+* SyncWrap::Geminabox - private gem server
+
+* SyncWrap::Qpid - AMQP broker
+
+=== Component wish list
+
+The above only represents my more recent project needs. Pull requests
+to add, for example, the following would be well received:
+
+* Bundler install, for example for Sinatra apps
+* Bluepill
+* Rails (extra credit: without git fetch or per-host asset compile)
+* Nginx
+* Mongo
+* Redis
 
-  SyncWrapper.new.define_tasks
+\SyncWrap makes it reasonable to implement any of the above in the
+privacy of your own office.
 
 == License
 
-Copyright (c) 2011-2013 David Kellum
+Copyright (c) 2011-2014 David Kellum
 
 Licensed under the Apache License, Version 2.0 (the "License"); you
 may not use this file except in compliance with the License.  You

data/Rakefile

@@ -6,68 +6,3 @@ require 'bundler/setup'
 require 'rjack-tarpit'
 
 RJack::TarPit.new( 'syncwrap' ).define_tasks
-
-require 'syncwrap/java'
-require 'syncwrap/hashdot'
-require 'syncwrap/jruby'
-require 'syncwrap/iyyov'
-require 'syncwrap/rhel'
-require 'syncwrap/aws'
-require 'syncwrap/postgresql'
-require 'syncwrap/remote_task'
-require 'syncwrap/geminabox'
-
-class SyncWrapper
-  include SyncWrap::Java
-  include SyncWrap::Hashdot
-  include SyncWrap::JRuby
-  include SyncWrap::Iyyov
-  include SyncWrap::RHEL
-  include SyncWrap::AWS
-  include SyncWrap::PostgreSQL
-  include SyncWrap::Geminabox
-
-  include SyncWrap::RemoteTask
-
-  def initialize
-    super
-
-    # SETUP: Install user@server instance goes here
-    set :domain, "localhost"
-
-  end
-
-  def define_tasks
-
-    desc "Combined Java, Hashdot, JRuby Deployment"
-    remote_task :jruby_deploy do
-      java_install
-      hashdot_install
-      jruby_install
-    end
-
-    desc "Deploy Iyyov Deamon"
-    remote_task :iyyov_deploy do
-      user_run_dir_setup
-      iyyov_install
-    end
-
-    desc "Deploy Geminabox Daemon"
-    remote_task :geminabox_deploy do
-      iyyov_install_jobs do
-        geminabox_install
-      end
-    end
-
-    desc "Deploy PostgreSQL"
-    remote_task :pg_deploy do
-      pg_install
-      pg_configure
-      pg_start
-    end
-
-  end
-
-end
-
-SyncWrapper.new.define_tasks

data/bin/syncwrap

@@ -0,0 +1,27 @@
+#!/usr/bin/env ruby
+# -*- ruby -*-
+
+#--
+# Copyright (c) 2011-2014 David Kellum
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you
+# may not use this file except in compliance with the License.  You
+# may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.  See the License for the specific language governing
+# permissions and limitations under the License.
+#++
+
+$LOAD_PATH.unshift File.expand_path( "../../lib", __FILE__ )
+
+require 'rubygems'
+
+require 'syncwrap'
+require 'syncwrap/cli'
+
+SyncWrap::CLI.new.run( ARGV )

data/examples/LAYOUT.rdoc

@@ -0,0 +1,70 @@
+# -*- coding: utf-8 -*-
+
+== Examples
+
+The examples/ directory contains several independent sync-file
+examples. Normally you would just have one named the default "sync.rb"
+Use the syncwrap command -f <file> option to load each.
+
+There is also an example Rakefile demonstrating basic rake integration.
+
+The sub-directories private/ and sync/ are standard named and shared
+by multiple examples.
+
+== Common Project Layout
+
+Here is what a complete end-user project setup might look like:
+
+  deploy/
+  ├── lib/
+  │   └── my_component.rb
+  ├── .gitignore
+  ├── private/
+  │   ├── aws.json
+  │   └── sec.pem
+  ├── sync.rb
+  └── sync/
+      ├── home
+      │   └── bob
+      │       └── .ssh
+      │           └── authorized_keys
+      └── my/
+          └── config.rb.erb
+
+sync.rb:: The aforementioned default named sync-file, containing
+          roles, components, hosts, and possibly profiles.
+
+sync/:: The default named sync root containing a tree of static files
+        and templates for pushing to hosts. See the rput.rb example
+        for usage.  The local sync root can contain overrides to the
+        syncwrap gem provided files/directories as well as new
+        files. Home directories and in particular
+        .ssh/authorized_keys is commonly used with the
+        SyncWrap::Users component.
+
+lib/:: Place project-specific components here. Once these are
+       non-trivial its nicer to `require_relative` them into the
+       sync.rb. Its best to namespace these in your own module.
+
+private/:: This directory is for sensitive files like credentials that
+           shouldn't be checked in. The files shown are for a typical
+           project using the SyncWrap::AmazonEC2 provider. See the
+           ec2.rb example. Similarly you can use a sync/private/ for
+           sensitive files/templates that will be deployed, and
+           include this as the first sync_root in your sync.rb You'll
+           have to externally arrange for sharing these files with
+           other users. Assuming you will check this into a git repo
+           (either its own, or for example, in a deploy/ directory of
+           a larger project repo), its a good idea to add a .gitignore
+           file containing the following at minimum:
+
+             private/
+
+           ...which ignores and avoids check-in by mistake of any so
+           named sub-directory and its contents.
+
+Without the private/ directories and sync.rb file, and adjusting lib/
+accordingly, you could also package this as a gem. This makes it easy
+to reuse components and files/templates across projects.  Include a
+bundler Gemfile in each deploy directory for precise version control
+of all gems involved, including syncwrap.

data/examples/Rakefile

@@ -0,0 +1,16 @@
+# -*- ruby -*-
+
+# Stupid simple Rakefile integration example.
+# See SyncWrap::Space#execute for additional customization options
+
+require 'rubygems'
+require 'bundler/setup'
+require 'syncwrap'
+
+desc "Execute default components on all hosts in SyncWrap hello.rb"
+task :deploy do
+  sp = SyncWrap::Space.new
+  sp.load_sync_file_relative 'hello.rb'
+  sp.default_options[:verbose] = true
+  sp.execute
+end

data/examples/ec2.rb

@@ -0,0 +1,44 @@
+# Demonstrates provisioning a PostgreSQL instance on Amazon EC2.
+#
+# This includes a custom 4 EBS-volume software RAID-1+0 for the
+# database storage.
+#
+# With API key-pair and pem (see SETUP below) create a host and
+# provision it via:
+#
+#   syncwrap -C postgres:pg-1 -v
+#
+# The host (name 'pg-1') definition will be automaticly added to the
+# end of this file.
+
+include SyncWrap
+
+# SETUP: edit the sample file for your API key-pair obtained via AWS console
+space.use_provider( AmazonEC2, aws_config: 'private/aws.json' )
+
+profile( :default,                  # The "default" profile
+         image_id: "ami-ccf297fc",  # Amazon Linux 2013.09.2 EBS 64 us-west-2
+         region: 'us-west-2',       # Oregon; or change region and ami.
+         user_data: :ec2_user_sudo, # Sudoer ec2-user \w no-tty required
+         key_name: 'sec',           # SETUP: Create this key in AWS console,
+                                    # or rename to an existing key.
+                                    # Same ssh_user_pem file below:
+         roles: [ :amazon_linux ] )
+
+profile( :postgres,                 # Inherits properties from :default
+         instance_type: 'm1.small',
+         ebs_volumes: 4,
+         ebs_volume_options: { size: 2 }, #gb
+         roles: [ :postgres ] )
+
+role( :amazon_linux,
+      Users.new( ssh_user: 'ec2-user',
+                 ssh_user_pem: 'private/sec.pem' ), # SETUP
+      RHEL.new,
+      Network.new )
+
+role( :postgres,
+      MDRaid.new( raw_devices: 4, lvm_volumes: [ [1.0, '/pg'] ] ),
+      PostgreSQL.new( pg_data_dir: '/pg/data' ) )
+
+# Generated Hosts:

data/examples/hello.rb

@@ -0,0 +1,14 @@
+
+class Greeter < SyncWrap::Component
+  def install
+    say_it
+  end
+
+  def say_it
+    sh <<-SH
+      echo "Hello from #{host.name}"
+    SH
+  end
+end
+
+host 'localhost', Greeter.new

data/examples/hello_binding.rb

@@ -0,0 +1,27 @@
+
+class GreetSupport < SyncWrap::Component
+
+  def install
+    # provision "echo" support, if that were needed.
+  end
+
+  def say( msg )
+    sh "echo '#{msg}'"
+  end
+
+end
+
+class Greeter < SyncWrap::Component
+
+  def install
+    say "Hello from #{host.name}"
+  end
+
+end
+
+host 'localhost', GreetSupport.new, Greeter.new
+
+# When executing in the context of localhost, this Greeter has access
+# to the public instance methods of this GreetSupport. The reverse is
+# not true: this GreetSupport instance does not get any Greeter
+# methods.

data/examples/jruby.rb

@@ -0,0 +1,11 @@
+include SyncWrap
+
+role( :all,
+      Users.new )
+
+role( :jruby,
+      OpenJDK.new,
+      JRubyVM.new( jruby_version: '1.7.10' ),
+      Hashdot.new )
+
+host( 'centos-1', RHEL.new, :jruby, internal_ip: '192.168.122.4' )

data/examples/private/aws.json

@@ -0,0 +1,4 @@
+/* SETUP: Replace with the real pem file */
+{ "access_key_id": "AKiiiiiiiiiiiiiiiiii",
+  "secret_access_key": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"
+}

data/examples/rput.rb

@@ -0,0 +1,24 @@
+# Important: Make this examples/sync directory first in
+# sync_paths. This is not done by default, to make sure its explicit.
+#
+# The second path is the SyncWrap gem's own sync directory.  The first
+# path can effectively override any file that would otherwise be found
+# in the second. This includes being able to override a static file
+# with a template, or the other way around. This is what we mean by
+# "transparent."
+space.prepend_sync_path
+
+class Writer < SyncWrap::Component
+  attr_accessor :message
+
+  def install
+    rput( "tmp/sample" ) # implied destination --> /tmp/sample
+  end
+end
+
+host 'localhost', Writer.new( message: "parameterized" )
+
+# Try changing the message property above and running repeatedly in
+# --verbose mode, to see how rsync uses checksums to detect
+# changes. The cryptic looking output lines are actually returned as
+# an array of changes from the rput call.

data/examples/sync/home/bob/.ssh/authorized_keys

@@ -0,0 +1 @@
+ssh-rsa AAAAB3Nzaxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx== bob@dot.net

data/examples/sync/tmp/sample.erb

@@ -0,0 +1,3 @@
+This could have been a simple file (i.e. 'sync/tmp/sample' ) but
+instead its a <%= message %> ERB template with access to Writer
+instance methods (including attributes).