dropsonde 0.0.2 → 0.0.6

data/lib/dropsonde/metrics.rb

@@ -1,86 +1,101 @@
+# frozen_string_literal: true
+
 require 'little-plugger'
+require 'securerandom'
 
+# metrics class
 class Dropsonde::Metrics
-  extend LittlePlugger( :path => 'dropsonde/metrics', :module => Dropsonde::Metrics)
+  extend LittlePlugger(path: 'dropsonde/metrics', module: Dropsonde::Metrics)
 
   def initialize
-    Dropsonde::Metrics.disregard_plugins(*Dropsonde.settings[:blacklist])
+    if Dropsonde.settings[:enable]
+      Dropsonde.settings[:disable] ||= []
+      disable = Dropsonde::Metrics.plugins.keys - Dropsonde.settings[:enable].map(&:to_sym)
+      Dropsonde.settings[:disable].concat disable
+    end
+
+    Dropsonde::Metrics.disregard_plugins(*Dropsonde.settings[:disable])
     Dropsonde::Metrics.initialize_plugins
   end
 
   def siteid
     return @siteid if @siteid
 
-    sha2 = Digest::SHA512.new
-    sha2.update Puppet.settings[:certname]
-    sha2.update Puppet.settings[:cacert]
-    sha2.update Dropsonde.settings[:seed] if Dropsonde.settings[:seed]
-    @siteid = sha2.hexdigest
+    @siteid = Dropsonde.settings[:siteid]
+
+    unless @siteid
+      sha2 = Digest::SHA512.new
+      sha2.update Puppet.settings[:certname]
+      sha2.update Puppet.settings[:cacert]
+      sha2.update Dropsonde.settings[:seed] if Dropsonde.settings[:seed]
+      @siteid = sha2.hexdigest
+    end
+
     @siteid
   end
 
   def list
-    str  = "                    Loaded telemetry plugins\n"
-    str << "                 ===============================\n\n"
+    str = "                    Loaded telemetry plugins\n"
+    str += "                 ===============================\n\n"
     Dropsonde::Metrics.plugins.each do |name, plugin|
-      str << name.to_s
-      str << "\n--------\n"
-      str << plugin.description.strip
-      str << "\n\n"
+      str += name.to_s
+      str += "\n--------\n"
+      str += plugin.description.strip
+      str += "\n\n"
     end
-    if Dropsonde.settings[:blacklist]
-      str << "Disabled plugins:\n"
-      str << "  #{Dropsonde.settings[:blacklist].join(', ')}"
+    if Dropsonde.settings[:disable]
+      str += "Disabled plugins:\n"
+      str += "  #{Dropsonde.settings[:disable].join(', ')}"
     end
     str
   end
 
   def schema
     schema = skeleton_schema
-    Dropsonde::Metrics.plugins.each do |name, plugin|
+    Dropsonde::Metrics.plugins.each do |_name, plugin|
       schema.concat(sanity_check_schema(plugin))
     end
     check_for_duplicates(schema)
     schema
   end
 
-  def preview
-    str  = "                      Puppet Telemetry Report Preview\n"
-    str << "                      ===============================\n\n"
-    Dropsonde::Metrics.plugins.each do |name, plugin|
+  def preview(puppetdb_session = nil)
+    str = "                      Puppet Telemetry Report Preview\n"
+    str += "                      ===============================\n\n"
+    Dropsonde::Metrics.plugins.each do |_name, plugin|
       schema = plugin.schema
 
-      plugin.setup
-      data = sanity_check_data(plugin)
-      plugin.cleanup
+      plugin.setup if plugin.respond_to? :setup
+      data = sanity_check_data(plugin, plugin.run(puppetdb_session))
+      plugin.cleanup if plugin.respond_to? :cleanup
 
-      str << plugin.name+"\n"
-      str << "-------------------------------\n"
-      str << plugin.description
+      str += "#{plugin.name}\n"
+      str += "-------------------------------\n"
+      str += plugin.description
       data.each do |row|
         key    = row.keys.first
-        values = row.values.first
+        values = row.values.flatten
 
-        desc = schema.find {|item| item[:name].to_sym == key.to_sym}[:description]
-        str << "- #{key}: #{desc}\n"
+        desc = schema.find { |item| item[:name].to_sym == key.to_sym }[:description]
+        str += "- #{key}: #{desc}\n"
         values.each do |item|
-          str << "    #{item}\n"
+          str += "    #{item}\n"
         end
       end
-      str << "\n\n"
+      str += "\n\n"
     end
-    str << "Site ID:\n"
-    str << siteid
+    str += "Site ID:\n"
+    str += siteid
     str
   end
 
-  def report
+  def report(puppetdb_session = nil)
     snapshots = {}
-    Dropsonde::Metrics.plugins.each do |name, plugin|
+    Dropsonde::Metrics.plugins.each do |_name, plugin|
       plugin.setup
-      sanity_check_data(plugin).each do |row|
+      sanity_check_data(plugin, plugin.run(puppetdb_session)).each do |row|
         snapshots[row.keys.first] = {
-          'value'     => row.values.first,
+          'value' => row.values.first,
           'timestamp' => Time.now.iso8601,
         }
       end
@@ -92,10 +107,31 @@ class Dropsonde::Metrics
     results
   end
 
-  def sanity_check_data(plugin)
-    data = plugin.run
-    keys_data   = data.map {|item| item.keys }.flatten.map(&:to_s)
-    keys_schema = plugin.schema.map {|item| item[:name] }
+  def example
+    require 'ipaddr'
+    results = skeleton_report
+    results[:message_id] = generate_guid
+    results[:timestamp]  = rand((Time.now - 60 * 60 * 24 * 365)..Time.now).utc
+    results[:ip]         = IPAddr.new(rand(2**32), Socket::AF_INET)
+    results.delete(:'self-service-analytics')
+
+    Dropsonde::Metrics.plugins.each do |_name, plugin|
+      sanity_check_data(plugin, plugin.example).each do |row|
+        results.merge!(row)
+      end
+    end
+
+    results
+  end
+
+  # We accept both the plugin and data gathered from the plugin so that
+  # we can sanitize both data and example data
+  def sanity_check_data(plugin, data)
+    # This allows plugin authors to easily skip metrics with no results
+    return [] if data.nil?
+
+    keys_data   = data.map { |item| item.keys }.flatten.map(&:to_s)
+    keys_schema = plugin.schema.map { |item| item[:name] }
 
     disallowed = (keys_data - keys_schema)
 
@@ -107,16 +143,16 @@ class Dropsonde::Metrics
   def sanity_check_schema(plugin)
     schema = plugin.schema
 
-    if schema.class != Array or schema.find {|item| item.class != Hash}
+    if (schema.class != Array) || schema.find { |item| item.class != Hash }
       raise "The #{plugin.name} plugin schema is not an array of hashes"
     end
 
     error = ''
     [:name, :type, :description].each do |field|
-      count = schema.reject {|item| item[field] }.count
-      next if count == 0
+      count = schema.reject { |item| item[field] }.count
+      next if count.zero?
 
-      error << "The #{plugin.name} plugin schema has #{count} missing #{field}s\n"
+      error += "The #{plugin.name} plugin schema has #{count} missing #{field}s\n"
     end
     raise error unless error.empty?
 
@@ -124,8 +160,8 @@ class Dropsonde::Metrics
   end
 
   def check_for_duplicates(schema)
-    keys  = schema.map {|col| col[:name] }
-    dupes = keys.select{ |e| keys.count(e) > 1 }.uniq
+    keys  = schema.map { |col| col[:name] }
+    dupes = keys.select { |e| keys.count(e) > 1 }.uniq
 
     raise "The schema defines duplicate keys: #{dupes}" unless dupes.empty?
   end
@@ -134,51 +170,55 @@ class Dropsonde::Metrics
     [
       {
         "description": "An ID that's unique for each checkin to Dujour.",
-        "mode": "NULLABLE",
-        "name": "message_id",
-        "type": "STRING"
+        "mode": 'NULLABLE',
+        "name": 'message_id',
+        "type": 'STRING',
       },
       {
-        "description": "A unique identifier for a site, derived as a hash of the CA certificate and optional seed.",
-        "mode": "NULLABLE",
-        "name": "site_id",
-        "type": "BYTES"
+        "description": 'A unique identifier for a site, derived as a hash of the CA certificate and optional seed.',
+        "mode": 'NULLABLE',
+        "name": 'site_id',
+        "type": 'BYTES',
       },
       {
-        "description": "The name of the product.",
-        "mode": "NULLABLE",
-        "name": "product",
-        "type": "STRING"
+        "description": 'The name of the product.',
+        "mode": 'NULLABLE',
+        "name": 'product',
+        "type": 'STRING',
       },
       {
-        "description": "Version of the project.",
-        "mode": "NULLABLE",
-        "name": "version",
-        "type": "STRING"
+        "description": 'Version of the project.',
+        "mode": 'NULLABLE',
+        "name": 'version',
+        "type": 'STRING',
       },
       {
-        "description": "Time the checkin to Dujour occurred.",
-        "mode": "NULLABLE",
-        "name": "timestamp",
-        "type": "TIMESTAMP"
+        "description": 'Time the checkin to Dujour occurred.',
+        "mode": 'NULLABLE',
+        "name": 'timestamp',
+        "type": 'TIMESTAMP',
       },
       {
-        "description": "IP Address of node checking in to Dujour.",
-        "mode": "NULLABLE",
-        "name": "ip",
-        "type": "STRING"
-      }
+        "description": 'IP Address of node checking in to Dujour.',
+        "mode": 'NULLABLE',
+        "name": 'ip',
+        "type": 'STRING',
+      },
     ]
   end
 
   def skeleton_report
     {
-      "product": "popularity-module",
-      "version": "1.0.0",
+      "product": 'popularity-module',
+      "version": '1.0.0',
       "site_id": siteid,
       "self-service-analytics": {
-        "snapshots": { }
-      }
+        "snapshots": {},
+      },
     }
   end
+
+  def generate_guid
+    SecureRandom.uuid
+  end
 end

data/lib/dropsonde/monkeypatches.rb

@@ -1,17 +1,18 @@
-module Puppet
-  class Module
+# frozen_string_literal: true
 
-    unless Module.method_defined? :"forge_module?"
-      def forge_module?
-        Dropsonde::Cache.forgeModule? self
-      end
+# puppet module class
+class Puppet::Module
+  unless Module.method_defined? :"forge_module?"
+    def forge_module?
+      Dropsonde::Cache.forge_module? self
     end
+  end
 
-    unless Module.method_defined? :forge_slug
-      def forge_slug
-        self.forge_name.tr('/','-') rescue nil
-      end
+  unless Module.method_defined? :forge_slug
+    def forge_slug
+      forge_name.tr('/', '-')
+    rescue StandardError
+      nil
     end
-
   end
 end

data/lib/dropsonde/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class Dropsonde
-  VERSION = '0.0.2'
+  VERSION = '0.0.6'
 end

data/lib/dropsonde.rb

@@ -1,23 +1,57 @@
+# frozen_string_literal: true
+
 require 'json'
 require 'httpclient'
 require 'puppetdb'
 require 'inifile'
+require 'puppet'
 
+# This class handles caching module process, generate reports,
+# fetchs all plugins defined in lib/dropsonde/metrics and also
+# handle connection and request to PuppetDB.
 class Dropsonde
   require 'dropsonde/cache'
   require 'dropsonde/metrics'
   require 'dropsonde/monkeypatches'
   require 'dropsonde/version'
 
-  @@pdbclient = nil
-  @@settings  = {}
+  def self.puppet_settings_overrides
+    overrides = []
+    if (confdir = ENV['PUPPET_CONFDIR'])
+      overrides << '--confdir'
+      overrides << confdir
+    end
+
+    if (codedir = ENV['PUPPET_CODEDIR'])
+      overrides << '--codedir'
+      overrides << codedir
+    end
+
+    if (vardir = ENV['PUPPET_VARDIR'])
+      overrides << '--vardir'
+      overrides << vardir
+    end
+
+    if (logdir = ENV['PUPPET_LOGDIR'])
+      overrides << '--logdir'
+      overrides << logdir
+    end
+
+    overrides
+  end
+
+  Puppet.initialize_settings(puppet_settings_overrides)
+
+  @pdbclient = nil
+  @settings  = {}
   def self.settings=(arg)
     raise "Requires a Hash to set all settings at once, not a #{arg.class}" unless arg.is_a? Hash
-    @@settings = arg
+
+    @settings = arg
   end
 
-  def self.settings
-    @@settings
+  class << self
+    attr_reader :settings
   end
 
   def self.generate_schema
@@ -29,24 +63,32 @@ class Dropsonde
     puts Dropsonde::Metrics.new.list
   end
 
-  def self.generate_report(format)
+  def self.generate_report(format, puppetdb_session = nil)
     case format
     when 'json'
-      puts JSON.pretty_generate(Dropsonde::Metrics.new.report)
+      puts JSON.pretty_generate(Dropsonde::Metrics.new.report(puppetdb_session))
     when 'human'
       puts
-      puts Dropsonde::Metrics.new.preview
+      puts Dropsonde::Metrics.new.preview(puppetdb_session)
     else
-      raise "unknown format"
+      raise 'unknown format'
     end
   end
 
   def self.submit_report(endpoint, port)
-    client = HTTPClient.new()
+    client = HTTPClient.new
+
+    # The httpclient gem ships with some expired CA certificates.
+    # This causes us to load the certs shipped with whatever
+    # Ruby is used to execute this gem's commands, which are generally
+    # more up-to-date, especially if using puppet-agent's Ruby.
+    #
+    # Note that this is no-op with Windows system Ruby.
+    client.ssl_config.set_default_paths
+
     result = client.post("#{endpoint}:#{port}",
-                  :header => {'Content-Type' => 'application/json'},
-                  :body   => Dropsonde::Metrics.new.report.to_json
-                )
+                         header: { 'Content-Type' => 'application/json' },
+                         body: Dropsonde::Metrics.new.report.to_json)
 
     if result.status == 200
       data = JSON.parse(result.body)
@@ -63,8 +105,18 @@ class Dropsonde
     end
   end
 
-  def self.puppetDB
-    return @@pdbclient if @@pdbclient
+  def self.generate_example(size, filename)
+    metrics = Dropsonde::Metrics.new
+    File.open(filename, 'w') do |file|
+      (0...size).each do |_i|
+        file.write(metrics.example.to_json)
+        file.write("\n")
+      end
+    end
+  end
+
+  def puppet_db
+    return @pdbclient if @pdbclient
 
     config = File.join(Puppet.settings[:confdir], 'puppetdb.conf')
 
@@ -72,14 +124,13 @@ class Dropsonde
 
     server = IniFile.load(config)['main']['server_urls'].split(',').first
 
-    @@pdbclient = PuppetDB::Client.new({
-      :server => server,
-      :pem    => {
-          'key'     => Puppet.settings[:hostprivkey],
-          'cert'    => Puppet.settings[:hostcert],
-          'ca_file' => Puppet.settings[:localcacert],
-      }
-    })
+    @pdbclient = PuppetDB::Client.new({
+                                        server: server,
+                                        pem: {
+                                          'key' => Puppet.settings[:hostprivkey],
+                                          'cert' => Puppet.settings[:hostcert],
+                                          'ca_file' => Puppet.settings[:localcacert],
+                                        },
+                                      })
   end
-
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dropsonde
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.6
 platform: ruby
 authors:
 - Ben Ford
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-07 00:00:00.000000000 Z
+date: 2021-11-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -153,13 +153,15 @@ files:
 - lib/dropsonde/cache.rb
 - lib/dropsonde/metrics.rb
 - lib/dropsonde/metrics/dependencies.rb
+- lib/dropsonde/metrics/environments.rb
 - lib/dropsonde/metrics/modules.rb
+- lib/dropsonde/metrics/platforms.rb
 - lib/dropsonde/metrics/puppetfiles.rb
 - lib/dropsonde/monkeypatches.rb
 - lib/dropsonde/version.rb
 homepage: https://github.com/puppetlabs/dropsonde
 licenses:
-- Apache 2
+- Apache-2.0
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -176,7 +178,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.6
+rubygems_version: 3.1.2
 signing_key: 
 specification_version: 4
 summary: A simple telemetry probe for gathering usage information about Puppet infrastructures.