kitchen_hooks 1.6.3 → 1.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 41cdc71627aae3524cbbb5642a0cf9364f5aa376
-  data.tar.gz: a6882ae0519cb298638dd1ab76007794e26bf563
+  metadata.gz: 57e367c61cc0d252a78fa6a7a934347f702b59bd
+  data.tar.gz: e3c96ae9eb5a51e064f968d564c7f90772ffd57e
 SHA512:
-  metadata.gz: 756af65131d98a742e2fb059368961e17a9ec1f0a4efce9ee2207770168c86e2fb162b658771facdddb739d3a686ce497c82be9a248211a768084086c8ec0174
-  data.tar.gz: 126a1a40db93e80992c4521cf5e032de4dd6613d8edaf0b4a85a47ed7c85e8e4cd21de206b8c53bd8137b8039942614ea8b52803402abe1dc71e59264d7d9647
+  metadata.gz: 0448e0bd79eddca8a029766ee64bfb6ce9767795492be28e13a91925c06b59ad1bfbe387d43f13097ffc4ddf6e01d06500db95e34124dd4c641a6222ebc7ab08
+  data.tar.gz: 37295349df30512542481063ddc17cf69a3c7fb0a171ddf7dcbe5bf664336fd1b72e4fe7420a5082fea59458b51e41dc071c1b9325a79efd17f0901116d85224

data/Readme.md

@@ -122,6 +122,7 @@ The `server` command also exposes some options for Sinatra configuration. See
 ### 1.6
 
 * First public release
+* Parallelize most operations
 * Add requests to backlog for serial processing
 
 #### 1.5

data/VERSION

@@ -1 +1 @@
-1.6.3
+1.7.0

data/etc/config.json

@@ -1,6 +1,7 @@
 {
   "hipchat": null,
   "knives": {
-    "user": "~/.chef/knife.rb"
+    "bng11": "~/.chef/bng11.rb",
+    "dc1": "~/.chef/dc1.rb"
   }
 }

data/kitchen_hooks.gemspec

@@ -17,7 +17,7 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'daybreak', '~> 0.3'
   s.add_runtime_dependency 'retryable', '~> 2'
   s.add_runtime_dependency 'berkshelf', '~> 3'
-  s.add_runtime_dependency 'chef', '~> 11.16.4'
+  s.add_runtime_dependency 'chef', '~> 12.0.0'
   s.add_runtime_dependency 'thor', '~> 0'
   s.add_runtime_dependency 'git', '~> 1.2'
   s.add_runtime_dependency 'sinatra', '~> 1.4'

data/lib/kitchen_hooks/app.rb

@@ -9,6 +9,8 @@ require 'sinatra/base'
 require_relative 'helpers'
 require_relative 'metadata'
 
+Thread.abort_on_exception = true
+
 
 module KitchenHooks
   class App < Sinatra::Application
@@ -23,6 +25,7 @@ module KitchenHooks
     def self.tmp! dir ; @@tmp = dir end
 
     def self.close!
+      @@sync_worker.kill
       @@backlog_worker.kill
       @@db.close
     end
@@ -37,7 +40,18 @@ module KitchenHooks
       end
     end
 
+    def self.sync!
+      @@sync_worker = Thread.new do
+        loop do
+          process_sync
+          sleep @@sync_interval
+        end
+      end
+    end
+
+
     def self.config! config
+      @@config = config
       @@hipchat = nil
       if config['hipchat']
         @@hipchat = HipChat::Client.new config['hipchat']['token']
@@ -47,6 +61,7 @@ module KitchenHooks
       @@knives = config['knives'].map do |_, knife|
         Pathname.new(knife).expand_path.realpath.to_s
       end
+      @@sync_interval = config.fetch 'sync_interval', 3600 # Hourly
     end
 
     get '/backlog' do
@@ -103,6 +118,7 @@ module KitchenHooks
       color = case entry[:type]
       when 'failure' ; 'red'
       when 'release' ; 'purple'
+      when 'unsynced' ; 'yellow'
       else ; 'green'
       end
       hipchat notification(entry), color
@@ -133,6 +149,20 @@ module KitchenHooks
     end
 
 
+    def self.process_sync
+      sync = sync_servers(knives).status
+
+      if sync.nil?
+        mark "Couldn't sync Chef servers (unknown issue)", 'unsynced'
+        return
+      end
+
+      sync_tag = sync[:num_failures].zero? ? 'synced' : 'unsynced'
+
+      mark sync, sync_tag
+    end
+
+
     def self.process event
       if event.nil? # JSON parse failed
         mark event, 'failure', 'Could not parse WebHook payload'

data/lib/kitchen_hooks/helpers.rb

@@ -10,12 +10,43 @@ require 'berkshelf'
 require 'sinatra/base'
 require 'pmap'
 
+require_relative 'helpers/sync_servers'
 
 Celluloid.logger = nil
 Berkshelf.logger = Logger.new $stdout
 
+
+
 module KitchenHooks
   class App < Sinatra::Application
+
+    def self.pluralize n, singular, plural=nil
+      plural = "#{singular}s" if plural.nil?
+      return "no #{plural}" if n.zero?
+      return "1 #{singular}" if n == 1
+      "#{n} #{plural}"
+    end
+
+    # http://stackoverflow.com/questions/4136248/how-to-generate-a-human-readable-time-range-using-ruby-on-rails
+    def self.humanize_seconds secs
+      [
+        [ 60, :seconds ],
+        [ 60, :minutes ],
+        [ 24, :hours ],
+        [ 1000, :days ]
+      ].map { |count, name|
+        if secs > 0
+          secs, n = secs.divmod(count)
+          "#{n.to_i} #{name}"
+        end
+      }.compact.reverse.join(' ')
+    end
+
+    def self.sync_servers knives
+      SyncServers.new knives
+    end
+
+
     def self.report_error e, msg=nil
       msg = e.message if msg.nil?
       $stdout.puts msg
@@ -304,7 +335,19 @@ module KitchenHooks
     def self.notification entry
       return entry[:error] if entry[:error]
       event = entry[:event]
+
       case entry[:type]
+      when 'synced', 'unsynced'
+        if event.is_a? String
+          event
+        else
+          'Synced <b>%d</b> of <b>%d</b> nodes (%s, %s elapsed)' % [
+            event[:num_successes],
+            event[:num_nodes],
+            pluralize(event[:num_failures], 'failure'),
+            humanize_seconds(event[:elapsed])
+          ]
+        end
       when 'kitchen upload'
         %Q| <i>#{author(event)}</i> updated <a href="#{gitlab_url(event)}">the Kitchen</a> |
       when 'cookbook upload'

data/lib/kitchen_hooks/helpers/sync_servers.rb

@@ -0,0 +1,64 @@
+require 'ridley'
+
+require 'set'
+
+
+class SyncServers
+  attr_reader :status
+
+
+  def initialize knives
+    @knives = knives
+    @started = Time.now
+    @status = sync_servers.merge \
+      elapsed: Time.now - @started
+  end
+
+
+private
+
+  def ridleys
+    @ridleys ||= @knives.map do |knife|
+      Ridley.from_chef_config(knife)
+    end
+  end
+
+  def all_nodes
+    @all_nodes ||= ridleys.flat_map do |ridley|
+      ridley.partial_search(:node, '*:*', %w[ ohai_time ])
+    end
+  end
+
+
+  def merged_nodes
+    @merged_nodes ||= all_nodes.group_by(&:name).pmap do |name, copies|
+      copies.sort_by { |c| c.automatic.ohai_time }.last
+    end
+  end
+
+
+  def sync_servers
+    nodes = merged_nodes
+    failures = Set.new
+
+    nodes.peach(8) do |n|
+      n.reload
+      ridleys.peach(4) do |ridley|
+        ridley.node.create(n) \
+        rescue ridley.node.update(n) \
+        rescue failures << n.name
+      end
+      puts 'Synced node "%s"' % n.name
+    end
+
+    return {
+      failures: failures,
+      num_successes: nodes.length - failures.length,
+      num_failures: failures.length,
+      num_nodes: nodes.length
+    }
+  rescue
+    return nil
+  end
+
+end

data/lib/kitchen_hooks/main.rb

@@ -61,10 +61,11 @@ module KitchenHooks
       desc: 'Location of temporary directory',
       default: '/tmp'
     def server
+      App.config! JSON::parse(File.read(options.config))
       App.backlog!
       App.db! options.database
       App.tmp! options.tmpdir
-      App.config! JSON::parse(File.read(options.config))
+      App.sync!
       App.set :environment, options.environment
       App.set :port, options.port
       App.set :bind, options.bind

data/web/app/style/kitchen_hooks.css

@@ -10,6 +10,10 @@ body {
   background: #d9534f;
 }
 
+.unsynced .cd-timeline-img {
+  background: #f0ad4e;
+}
+
 .release .cd-timeline-img {
   background: #5bc0de;
 }

data/web/public/vendor/img/sync.svg

@@ -0,0 +1,12 @@
+<?xml version="1.0"?>
+<svg width="800" height="800" xmlns="http://www.w3.org/2000/svg">
+ <g>
+  <title>background</title>
+  <rect x="-1" y="-1" width="802" height="802" id="canvas_background" fill="none"/>
+ </g>
+
+ <g>
+  <title>Layer 1</title>
+  <path id="cloud-5-icon" d="m709.958801,397.394226c-12.598816,-158.646255 -221.498688,-210.920303 -306.884827,-75.072937c-55.378632,-31.311005 -126.010834,5.526672 -130.436066,70.253632c-60.718811,7.439392 -107.756332,59.146332 -107.756332,121.873932c0,67.83905 54.990204,122.829346 122.829361,122.829346l384.973877,0c67.835327,0 122.827454,-54.990295 122.827454,-122.829346c0.002014,-54.838409 -35.941895,-101.266602 -85.553467,-117.054626zm-591.126747,143.131042c-65.176666,-6.489807 -115.318749,-60.720673 -115.318749,-126.443329c0,-57.07019 37.981427,-106.291473 91.172333,-121.91629c14.955742,-50.987915 68.608047,-80.878265 119.807449,-67.225845c30.647736,-36.243652 75.486191,-57.308533 123.855881,-57.308533c44.846069,0 85.705231,18.275543 115.236084,47.994858c-17.300964,7.968079 -33.211975,18.15062 -47.62381,30.420929c-19.500092,-15.799637 -43.733093,-23.289108 -67.612274,-23.289108c-69.332733,0 -97.812119,52.479721 -104.920914,72.575729c-28.264053,-40.766861 -104.982407,-9.986542 -89.084747,44.580872c-47.595016,-8.629364 -85.703339,27.202789 -85.703339,74.167389c0,35.530396 25.891945,65.111328 60.782272,70.918701c-3.256432,18.646515 -3.39489,37.287445 -0.590187,55.524628z" fill="#ffffff"/>
+ </g>
+</svg>

data/web/views/app.erb

@@ -16,6 +16,10 @@
           <img src="/vendor/img/error.svg" alt="Error">
         <% when 'release' %>
           <img src="/vendor/img/cube.svg" alt="Cube">
+        <% when 'synced' %>
+          <img src="/vendor/img/sync.svg" alt="Sync">
+        <% when 'unsynced' %>
+          <img src="/vendor/img/sync.svg" alt="Sync">
         <% end %>
       </div>
       <div class="cd-timeline-content">
@@ -26,6 +30,10 @@
           <p><%= notification entry %></p>
         <% elsif type == 'release' %>
           <p><%= notification entry %></p>
+        <% elsif type == 'synced' %>
+          <p><%= notification entry %></p>
+        <% elsif type == 'unsynced' %>
+          <p><%= notification entry %></p>
         <% else %>
           <p><%= notification entry %> by pushing <%= push_details(entry[:event]) %></p>
         <% end %>

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kitchen_hooks
 version: !ruby/object:Gem::Version
-  version: 1.6.3
+  version: 1.7.0
 platform: ruby
 authors:
 - Sean Clemmer
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-04 00:00:00.000000000 Z
+date: 2015-02-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: hipchat
@@ -72,14 +72,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 11.16.4
+        version: 12.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 11.16.4
+        version: 12.0.0
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
@@ -171,6 +171,7 @@ files:
 - lib/kitchen_hooks.rb
 - lib/kitchen_hooks/app.rb
 - lib/kitchen_hooks/helpers.rb
+- lib/kitchen_hooks/helpers/sync_servers.rb
 - lib/kitchen_hooks/main.rb
 - lib/kitchen_hooks/metadata.rb
 - web/app/kitchen_hooks.js
@@ -183,6 +184,7 @@ files:
 - web/public/vendor/img/cutlery.svg
 - web/public/vendor/img/error.svg
 - web/public/vendor/img/shield.svg
+- web/public/vendor/img/sync.svg
 - web/public/vendor/js/bootstrap-v3.3.0.js
 - web/public/vendor/js/jquery-timeago-v1.4.1.js
 - web/public/vendor/js/jquery-v2.1.1.js