puppetdb_rundeck 0.3.1 → 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5ae36fec2f31239f863f2c1c7622631234342377
+  data.tar.gz: 37029c01975875609d3ede38bdecf9023f74e706
+SHA512:
+  metadata.gz: bd5bb497c3f595ab1cc08bbfea46746f01ec536451db869d4d85a038d24bbd504b3e69490287df0fb42656ca29080fd6dddc4854652149ada48fe8b82115efda
+  data.tar.gz: de80570a32684aefd7cd1e5d1dfad38d8f24b448416cf875aca6670a176d5ee34abebaaca262ec14f8ca83ad7ee86f5be236e7d3aa3a1a6b3a71fa406457113a

data/CHANGELOG

@@ -1,6 +1,10 @@
+2014-11-19 Release 1.0.0
+  - fixing some threading bugs
+  - excluding some facts that can cause issues
+
 2014-07-01 Release 0.3.1
   - fixing issue when running on 1.8.x
-  
+
 2014-07-01 Release 0.3.0
   - improved performance for large instances of puppetdb
   - added 30 min cache with api for clearing cache

data/bin/puppetdb_rundeck

@@ -21,6 +21,14 @@ OptionParser.new do |opts|
   opts.on("--pdbport PORT", String, "The port of the PuppetDB instance") do |pp|
     options[:pdbport] = pp
   end
+
+  opts.on("--cache-timeout TIMEOUT", Integer, "The number of seconds to cache the results from puppetdb") do |ct|
+    options[:cache_timeout] = ct
+  end
+
+  opts.on("--thread-count THREADS", Integer, "Limit the number of threads used by the application when communicating with PuppetDB") do |tc|
+    options[:thread_count] = tc
+  end
 end.parse!
 
 if options[:pdbhost].nil? or options[:pdbhost].eql?('')
@@ -36,6 +44,8 @@ end
 PuppetDBRunDeck.set :bind, '0.0.0.0'
 PuppetDBRunDeck.set :puppetdb_host, options[:pdbhost]
 PuppetDBRunDeck.set :puppetdb_port, options[:pdbport]
+PuppetDBRunDeck.set :cache_timeout, options[:cache_timeout]
+PuppetDBRunDeck.set :thread_count, options[:thread_count]
 
 if options[:port].nil? or options[:port].eql?('')
   PuppetDBRunDeck.set :port, '4567'
@@ -43,4 +53,16 @@ else
   PuppetDBRunDeck.set :port, options[:port]
 end
 
-PuppetDBRunDeck.run!
+if options[:cache_timeout].nil? or options[:cache_timeout].eql?('')
+  PuppetDBRunDeck.set :cache_timeout, 30
+else
+  PuppetDBRunDeck.set :cache_timeout, options[:cache_timeout]
+end
+
+if options[:thread_count].nil? or options[:thread_count].eql?('')
+  PuppetDBRunDeck.set :thread_count, 3
+else
+  PuppetDBRunDeck.set :thread_count, options[:thread_count]
+end
+
+PuppetDBRunDeck.run!

data/lib/app.rb

@@ -13,7 +13,7 @@ class PuppetDBRunDeck < Sinatra::Base
   def initialize(app = nil, params = {})
     super(app)
     puppetdb_helper = Helpers::PuppetDB.new(settings.puppetdb_host, settings.puppetdb_port)
-    @endpoint = EndPoint.new(puppetdb_helper)
+    @endpoint = EndPoint.new(puppetdb_helper, settings.cache_timeout, settings.thread_count)
   end
 
   get '/' do
@@ -26,17 +26,17 @@ class PuppetDBRunDeck < Sinatra::Base
 
   get '/api/yaml' do
     content_type 'text/yaml'
-    content = @endpoint.to_yaml()
+    content = @endpoint.to_yaml(true)
   end
 
   get '/api/xml' do
     content_type 'application/xml'
-    content = @endpoint.to_xml()
+    content = @endpoint.to_xml(true)
   end
 
   get '/api/json' do
     content_type 'application/json'
-    content = @endpoint.to_json()
+    content = @endpoint.to_json(true)
   end
 
   get '/cache' do

data/lib/helpers/process.rb

@@ -1,51 +1,45 @@
 
 class Helpers::Process
 
-  def add_facts(facts, host, collection)
+  def add_facts(facts, host)
+
+    fact_collection = { host => {} }
     facts.each{|f|
       if f['certname'].eql?(host)
         fact_name = f['name']
         fact_value = f['value']
 
-        if fact_name.include?('processor')
-          fact_name = number_to_name(fact_name)
+        if fact_name.include?('hostname')
+          fact_value = host
         end
 
-        if fact_name.include?('path')
-          fact_value = fact_value.gsub!('"','')
+        if !is_excluded?(fact_name)
+          fact_collection[host][fact_name] = fact_value
         end
 
-        if collection.instance_of?(Hash)
-          if collection[host].nil?
-            collection[host] = {}
-          end
-
-          collection[host][fact_name] = fact_value
-
-        elsif collection.instance_of?(Array)
-          collection << "#{fact_name}=\"#{fact_value}\" "
-        else
-        end
       end
     }
-    return collection
+    return fact_collection
   end
 
-  private
-    def number_to_name(name)
-      lookup = {
-        '1' => 'one', '2' => 'two', '3' => 'three', '4' => 'four', '5' => 'five',
-        '6' => 'six', '7' => 'seven', '8' => 'eight', '9' => 'nine', '10' => 'ten',
-        '11' => 'eleven', '12' => 'twelve', '13' => 'thirteen', '14' => 'fourteen',
-        '15' => 'fifteen', '16' => 'sixteen'
-      }
-
-      lookup.each { |k,v|
-        if name.include?(k)
-          name.gsub!(k,"_#{v}")
-        end
-      }
-      return name
+  def is_excluded?(fact)
+
+    excluded_facts = [
+      '^processor(s|\d+)$', '^path$', '^utc_offset$', '^os$',
+      '^ec2_metrics_vhostmd$', '^ec2_network_interfaces_macs.*',
+      '^ec2_userdata$', '^ec2_metadata$',
+      '^partitions$', '^system_uptime$', '^apt_package_updates$'
+    ]
+
+    match = false
+    for ex in excluded_facts
+      if fact.match(ex)
+        match = true
+        break;
+      end
     end
 
+    return match
+  end
+
 end

data/lib/helpers/puppetdb.rb

@@ -32,9 +32,7 @@ module Helpers
       else
         fact_endpoint = "http://#{@puppetdb_host}:#{@puppetdb_port}/v3/facts"
       end
-
-      #p "requesting facts by: #{fact_endpoint}"
-
+      
       uri = URI.parse(fact_endpoint)
       request = Net::HTTP::Get.new(uri.path)
       request.add_field('Accept', 'application/json')

data/lib/model/endpoint.rb

@@ -1,17 +1,23 @@
 require File.expand_path('../../helpers/process', __FILE__)
+require 'thread'
+require 'thwait'
 
 class EndPoint
 
   attr_accessor :cache_timeout
-  THREAD_COUNT = 40
+  attr_accessor :thread_count
 
-  def initialize(puppetdb_helper)
+  def initialize(puppetdb_helper, cache_timeout, thread_count)
     @db_helper = puppetdb_helper
-    @cache_timeout = 1800
+    @cache_timeout = cache_timeout
+    @thread_count = thread_count
   end
 
   def reload(type)
-    nodes = @db_helper.get_nodes
+    p "reloading data"
+    if @nodes.nil? or @nodes.empty?
+      @nodes = @db_helper.get_nodes
+    end
 
     @rundeck_data = Hash.new
 
@@ -21,23 +27,36 @@ class EndPoint
 
     per_type_cache = "/tmp/puppetdb-resource.#{type}"
 
-    THREAD_COUNT.times.map {
-      Thread.new(nodes) do |nodes|
-        while node = mutex.synchronize { nodes.pop }
+    data_elements = []
+    process_threads = []
+
+    @thread_count.times.map {
+      t = Thread.new(@nodes) do |nodes|
+        while node = mutex.synchronize { @nodes.pop }
           host = node['name']
           facts = @db_helper.get_facts(host)
           if !facts.nil?
-            mutex.synchronize { helper.add_facts(facts, host, @rundeck_data) }
+            mutex.synchronize do
+              data_elements.push(helper.add_facts(facts, host))
+            end
           end
         end
       end
-    }.each(&:join)
+      process_threads.push(t)
+    }
+
+    ThreadsWait.all_waits(*process_threads)
+
+    data_elements.each do |item|
+      #sleep(Random.new.rand(1..10))
+      @rundeck_data.merge!(item) if @rundeck_data.is_a?(Hash)
+    end
 
     data = case type
-      when 'json' then to_json(false)
-      when 'yaml' then to_yaml(false)
+      when 'json' then self.to_json(false)
+      when 'yaml' then self.to_yaml(false)
       when 'xml' then to_xml(false)
-      else "unknown"
+      else 'unknown'
     end
 
     File.open(per_type_cache, 'w') { |file| file.write(data) }
@@ -45,7 +64,8 @@ class EndPoint
     return data
   end
 
-  def to_json(parse_data=true)
+  def to_json(parse_data)
+    p "parse_data is: #{parse_data}"
     parse('json') if parse_data == true
     if @rundeck_data.is_a?(String)
       @rundeck_data
@@ -54,7 +74,8 @@ class EndPoint
     end
   end
 
-  def to_yaml(parse_data=true)
+  def to_yaml(parse_data)
+    p "parse_data is: #{parse_data}"
     parse('yaml') if parse_data == true
     if @rundeck_data.is_a?(String)
       @rundeck_data
@@ -64,8 +85,6 @@ class EndPoint
   end
 
   def to_xml(parse_data=true)
-    helper = Helpers::Process.new
-
     parse('xml') if parse_data == true
     if @rundeck_data.is_a?(String)
       @rundeck_data
@@ -86,7 +105,6 @@ class EndPoint
     end
   end
 
-
   def parse(type)
     per_type_cache = "/tmp/puppetdb-resource.#{type}"
 
@@ -109,7 +127,7 @@ class EndPoint
   end
 
   def clear_cache
-    cache_files = ['/tmp/puppetdb-resource.xml','/tmp/puppetdb-resource.json','/tmp/puppetdb-resource.yaml']
+    cache_files = ['/tmp/puppetdb-resource.json','/tmp/puppetdb-resource.yaml']
 
     cache_files.each do |file|
       if File.exist?(file)

data/lib/views/api.haml

@@ -2,6 +2,8 @@
 %html
   %head
     %title PuppetDB-Rundeck
+    %link(rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.2.0/css/bootstrap.min.css")
+    %link(rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.2.0/css/bootstrap-theme.min.css")
   %body
     %h1 PuppetDB-Rundeck
     %p

data/lib/views/cache.haml

@@ -2,6 +2,8 @@
 %html
   %head
     %title PuppetDB-Rundeck
+    %link(rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.2.0/css/bootstrap.min.css")
+    %link(rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.2.0/css/bootstrap-theme.min.css")
   %body
     %h1 PuppetDB-Rundeck
     %h2 Cache settings

data/puppetdb_rundeck.gemspec

@@ -3,12 +3,12 @@ $:.push File.expand_path("../lib", __FILE__)
 
 Gem::Specification.new do |spec|
   spec.name          = 'puppetdb_rundeck'
-  spec.version       = '0.3.1'
+  spec.version       = '1.0.0'
   spec.authors       = ['liamjbennett']
   spec.email         = ['lbennett@opentable.com']
   spec.summary       = %q{A sinatra based application to provide integration between PuppetDB and Rundeck}
   spec.description   = %q{puppetdb_rundeck is a sinatra based application to provide integration between PuppetDB and Rundeck.
-It provides an api in either xml or yaml that allows the node and fact data with puppetdb to use used as a resource
+It provides an api in either xml, json or yaml that allows the node and fact data with puppetdb to use used as a resource
 with rundeck}
   spec.homepage      = 'https://github.com/opentable/puppetdb_rundeck'
   spec.license       = 'MIT'

metadata

@@ -1,20 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: puppetdb_rundeck
 version: !ruby/object:Gem::Version
-  version: 0.3.1
-  prerelease: 
+  version: 1.0.0
 platform: ruby
 authors:
 - liamjbennett
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-07-01 00:00:00.000000000 Z
+date: 2014-11-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -22,7 +20,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -30,7 +27,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -38,7 +34,6 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -46,87 +41,76 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rack-test
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec-mocks
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: builder
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: json_pure
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -134,7 +118,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -142,7 +125,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: tilt
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -150,7 +132,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -158,7 +139,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: sinatra
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -166,7 +146,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -174,7 +153,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: haml
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -182,7 +160,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -190,7 +167,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: thread
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -198,18 +174,14 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.1'
-description: ! 'puppetdb_rundeck is a sinatra based application to provide integration
-  between PuppetDB and Rundeck.
-
-  It provides an api in either xml or yaml that allows the node and fact data with
-  puppetdb to use used as a resource
-
-  with rundeck'
+description: |-
+  puppetdb_rundeck is a sinatra based application to provide integration between PuppetDB and Rundeck.
+  It provides an api in either xml, json or yaml that allows the node and fact data with puppetdb to use used as a resource
+  with rundeck
 email:
 - lbennett@opentable.com
 executables:
@@ -242,27 +214,26 @@ files:
 homepage: https://github.com/opentable/puppetdb_rundeck
 licenses:
 - MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23.2
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: A sinatra based application to provide integration between PuppetDB and Rundeck
 test_files:
 - spec/app_spec.rb