pupistry 0.0.10 → 0.0.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e5f9601622da2e9d834ecb81aec0b908ef5ad12e
-  data.tar.gz: 2fdd67a6cb8433ee0556e24b3c17ee0836657849
+  metadata.gz: b516bef9aa8033e4871098cd224fe497ef82f1b6
+  data.tar.gz: 7b158bdc381764e80618a4236e6287d19ac6670e
 SHA512:
-  metadata.gz: f13aeefb955e9784a5c1b34d861ef6ff2f6d7c94e89ac0f1c0d0bcb7f28409bc2b3e76a62786da160773d872bc229b5b52715c08fbecae8dea7f3b713254664b
-  data.tar.gz: 044fd9496533f320be7f0a0e17963b8713a5ca00f70408ae0858587005a41e4ea1c74b71b3bec8d2eaf7132afdf821499fd1d806a145abb1f70de706981b5217
+  metadata.gz: 8f2f7720b7b9ef5f8042282d49ba3093bfcc4c160bbc8288ba200410daaae4da4842d4b2bc21059879adb0ac4686a841413a58f4be2f90c9ba37a980a3ec5aed
+  data.tar.gz: 3466558b948d2e141fb73a1b7e86ac898dc4d88504133d8b450c14124b1013e6ec42a8f0d421eeacb80904e7e3960b79fd94ed1a31f2df6cb835fabecf898dd7

data/bin/pupistry

@@ -49,105 +49,18 @@ class CLI < Thor
       $logger.warn "A daemon running in noop will do nothing except log what changes it could apply"
     end
 
-    # Muppet dev check
-    if options[:daemon]
-      $logger.fatal "Daemon mode not implemented yet"
-      exit 0
-    end
-
-
-    # Fetch artifact versions
-    $logger.info "Checking version of artifact available..."
-
-    artifact = Pupistry::Artifact.new
-    artifact.checksum = artifact.fetch_latest
-
-    unless artifact.checksum
-      $logger.error "There is no current artifact available for download, no steps can be taken."
-      exit 0
-    end
-
-    artifact_installed = Pupistry::Artifact.new
-    artifact_installed.checksum = artifact_installed.fetch_installed
-
-    if artifact_installed.checksum
-      $logger.debug "Currently on #{artifact_installed.checksum}"
-    else
-      $logger.debug "No currently installed artifact - blank slate!"
-    end
-
-    # Download the new artifact if one has changed. If we already have this
-    # version, then we should skip downloading and go straight to running
-    # Puppet - unless the user runs with --force (eg to fix a corrupted
-    # artifact).
-    
-    if artifact.checksum != artifact_installed.checksum or options[:force]
-      if options[:force]
-        $logger.warn "Forcing download of latest artifact regardless of current one."
-      end
-
-      # Install the artifact
-      $logger.info "Downloading latest artifact (#{artifact.checksum})..."
-
-      artifact.fetch_artifact
-      artifact.unpack
-
-      unless artifact.install
-        $logger.fatal "An unexpected error happened when installing the latest artifact, cancelling Puppet run"
-        exit 0
-      end
-
-      # Remove temporary unpacked files
-      artifact.clean_unpack
-    else
-      $logger.info "Already have latest artifact applied."
-
-      # By default we run Puppet even if we have the latest artifact. There's
-      # some grounds for debate about whether this is the right thing - in some
-      # ways it is often a waste of CPU, since if the artifact hasn't changed,
-      # then it's unlikley anything else has changed.
-      #
-      # But that's not always 100% true - Puppet will undo local changes or
-      # upgrade package versions (ensure => latest) if appropiate, so we should
-      # act like the standard command and attempt to apply whatever we can.
-      #
-      # To provide users with options, we provide the --lazy parameter to avoid
-      # running Puppet except when the artifact changes. By default, Puppet
-      # runs every thing to avoid surprise.
-      
-      if options[:minimal]
-        $logger.info "Running with minimal effort mode enabled, not running Puppet since artifact version already applied"
-        exit 0
-      end
-
+    if options[:force] and options[:daemon]
+      $logger.warn "A daemon running with force will be very wasteful of system resources! NOT RECOMMENDED."
     end
 
 
-    # Execute Puppet.
-    puppet_cmd = "puppet apply"
-
-    if options[:noop]
-      puppet_cmd += " --noop"
-    end
-
-    if options[:environment]
-      environment = options[:environment]
+    if options[:daemon]
+      # Run as a daemon service
+      Pupistry::Agent.daemon options
     else
-      environment = 'master'
+      # Single-run Agent Execution
+      Pupistry::Agent.apply options
     end
-
-    puppet_cmd += " --environment #{environment}"
-    puppet_cmd += " --modulepath #{$config["agent"]["puppetcode"]}/#{environment}/modules/"
-    puppet_cmd += " --hiera_config #{$config["agent"]["puppetcode"]}/#{environment}/hiera.yaml"
-    puppet_cmd += " #{$config["agent"]["puppetcode"]}/#{environment}/manifests/site.pp"
-
-    $logger.info "Executing Puppet..."
-    $logger.debug "With: #{puppet_cmd}"
-
-    unless system puppet_cmd
-      $logger.fatal "An unexpected issue occured when running puppet"
-    end
-
   end
 
 
@@ -243,6 +156,11 @@ class CLI < Thor
     #
     # So given this, we might as well go native and just rely on the system
     # diff command to do the job.
+    #
+    # TODO: We need smarts here to handle git branching, so a branch doens't
+    #       produce a new mega diff, we want only the real changes to be
+    #       easily visible, or the diff function loses value to people.
+    #       Pull requests welcome :-) xoxo
     
     Dir.chdir("#{$config["general"]["app_cache"]}/artifacts/") do
       unless system "diff -Nuar unpacked.#{artifact_upstream.checksum} unpacked.#{artifact_current.checksum}"

data/lib/pupistry/agent.rb

@@ -0,0 +1,173 @@
+require 'rubygems'
+require 'fileutils'
+require 'rufus/scheduler'
+
+module Pupistry
+  # Pupistry::Agent
+
+  class Agent
+    # Functions for running the Pupistry agent aka "apply mode" to actually
+    # download and run Puppet against the contents of the artifact.
+
+
+    def self.daemon options
+      ## Run as a daemon
+
+
+      # Since options comes from Thor, it can't be modified, so we need to
+      # copy the options and then we can edit it.
+
+      options_new = options.inject({}) do |new, (name, value)| 
+        new[name] = value;
+        new 
+      end
+
+      # If the minimal mode has been enabled in config, respect.
+      if $config["agent"]["daemon_minimal"]
+        options_new[:minimal] = true
+      end
+
+      # If no frequency supplied, use 300 seconds safe default.
+      unless $config["agent"]["daemon_frequency"]
+        $config["agent"]["daemon_frequency"] = 300
+      end
+
+
+      # Use rufus-scheduler to run our apply job as a regularly scheduled job
+      # but with build in locking handling.
+      
+      $logger.info "Launching daemon... frequency of #{$config["agent"]["daemon_frequency"]} seconds."
+
+      begin
+
+        scheduler = Rufus::Scheduler.new
+
+        scheduler.every "#{$config["agent"]["daemon_frequency"]}s", :overlap => false, :timeout => '1d', :first_at => Time.now + 1 do
+          $logger.info "Triggering another Pupistry run (#{$config["agent"]["daemon_frequency"]}s)"
+          apply options_new
+        end
+
+        scheduler.join
+
+      rescue Rufus::Scheduler::TimeoutError
+        $logger.error "A run of Pupistry timed out after 1 day as a safety measure. There may be a bug or a Puppet action causing it to get stuck"
+
+      rescue SignalException => e
+        # Clean shutdown signal (eg SIGTERM)
+        $logger.info "Clean shutdown of Pupistry daemon requests"
+        exit 0
+
+      rescue Exception => e
+        raise e
+      end
+
+    end
+
+
+    def self.apply options
+      ## Download and apply the latest artifact (if any)
+
+      # Fetch artifact versions
+      $logger.info "Checking version of artifact available..."
+
+      artifact = Pupistry::Artifact.new
+      artifact.checksum = artifact.fetch_latest
+
+      unless artifact.checksum
+        $logger.error "There is no current artifact available for download, no steps can be taken."
+        return false
+      end
+
+      artifact_installed = Pupistry::Artifact.new
+      artifact_installed.checksum = artifact_installed.fetch_installed
+
+      if artifact_installed.checksum
+        $logger.debug "Currently on #{artifact_installed.checksum}"
+      else
+        $logger.debug "No currently installed artifact - blank slate!"
+      end
+
+      # Download the new artifact if one has changed. If we already have this
+      # version, then we should skip downloading and go straight to running
+      # Puppet - unless the user runs with --force (eg to fix a corrupted
+      # artifact).
+      
+      if artifact.checksum != artifact_installed.checksum or options[:force]
+        if options[:force]
+          $logger.warn "Forcing download of latest artifact regardless of current one."
+        end
+
+        # Install the artifact
+        $logger.info "Downloading latest artifact (#{artifact.checksum})..."
+
+        artifact.fetch_artifact
+        artifact.unpack
+
+        unless artifact.install
+          $logger.fatal "An unexpected error happened when installing the latest artifact, cancelling Puppet run"
+          return false
+        end
+
+        # Remove temporary unpacked files
+        artifact.clean_unpack
+      else
+        $logger.info "Already have latest artifact applied."
+
+        # By default we run Puppet even if we have the latest artifact. There's
+        # some grounds for debate about whether this is the right thing - in some
+        # ways it is often a waste of CPU, since if the artifact hasn't changed,
+        # then it's unlikley anything else has changed.
+        #
+        # But that's not always 100% true - Puppet will undo local changes or
+        # upgrade package versions (ensure => latest) if appropiate, so we should
+        # act like the standard command and attempt to apply whatever we can.
+        #
+        # To provide users with options, we provide the --lazy parameter to avoid
+        # running Puppet except when the artifact changes. By default, Puppet
+        # runs every thing to avoid surprise.
+        
+        if options[:minimal]
+          $logger.info "Running with minimal effort mode enabled, not running Puppet since artifact version already applied"
+          return false
+        end
+
+      end
+
+      # Check if the requested environment/branch actually exists
+      if options[:environment]
+        environment = options[:environment]
+      else
+        environment = 'master'
+      end
+
+      unless Dir.exists?("#{$config["agent"]["puppetcode"]}/#{environment}")
+        $logger.fatal "The requested branch/environment of #{environment} does not exist, unable to run Puppet"
+        return false
+      end
+
+
+      # Execute Puppet.
+      puppet_cmd = "puppet apply"
+
+      if options[:noop]
+        puppet_cmd += " --noop"
+      end
+
+      puppet_cmd += " --environment #{environment}"
+      puppet_cmd += " --modulepath #{$config["agent"]["puppetcode"]}/#{environment}/modules/"
+      puppet_cmd += " --hiera_config #{$config["agent"]["puppetcode"]}/#{environment}/hiera.yaml"
+      puppet_cmd += " #{$config["agent"]["puppetcode"]}/#{environment}/manifests/site.pp"
+
+      $logger.info "Executing Puppet..."
+      $logger.debug "With: #{puppet_cmd}"
+
+      unless system puppet_cmd
+        $logger.error "An unexpected issue occured when running puppet"
+      end
+
+    end
+
+  end 
+end
+
+# vim:shiftwidth=2:tabstop=2:softtabstop=2:expandtab:smartindent

data/lib/pupistry/bootstrap.rb

@@ -72,6 +72,8 @@ module Pupistry
         secret_access_key: $config["agent"]["secret_access_key"],
         region: $config["agent"]["region"],
         proxy_uri: $config["agent"]["proxy_uri"],
+        daemon_frequency: $config["agent"]["daemon_frequency"],
+        daemon_minimal: $config["agent"]["daemon_minimal"]
       }
 
       # Generate template using ERB

data/lib/pupistry.rb

@@ -1,3 +1,4 @@
+require 'pupistry/agent'
 require 'pupistry/artifact'
 require 'pupistry/bootstrap'
 require 'pupistry/config'

data/settings.example.yaml

@@ -47,6 +47,19 @@ agent:
   region: ap-southeast-2
   proxy_uri:
 
+  # (If Daemonised)
+  # Default is to check for a new artifact every 60 seconds, but only to
+  # actually run Puppet if there has been a change to the artifact contents.
+  # 
+  # At a polling rate of 60 seconds, the cost of S3 will be about $0.02 per
+  # month per system running Pupistry.
+  #
+  # If you want to force regular Puppet runs regardless whether or not a new
+  # artifact has been released, turn daemon_minimal off, but make sure the
+  # frequency isn't too low - 300 seconds+ recommended otherwise Puppet will
+  # be hammering your system resources.
+  daemon_frequency: 60
+  daemon_minimal: true
 
 
 # The following settings are only needed on the build machines (people building

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: pupistry
 version: !ruby/object:Gem::Version
-  version: 0.0.10
+  version: 0.0.11
 platform: ruby
 authors:
 - Jethro Carr
@@ -52,6 +52,20 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: rufus-scheduler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3'
 - !ruby/object:Gem::Dependency
   name: r10k
   requirement: !ruby/object:Gem::Requirement
@@ -75,6 +89,7 @@ extra_rdoc_files: []
 files:
 - bin/pupistry
 - lib/pupistry.rb
+- lib/pupistry/agent.rb
 - lib/pupistry/artifact.rb
 - lib/pupistry/bootstrap.rb
 - lib/pupistry/config.rb