dropsonde 0.0.5 → 0.0.8

data/lib/dropsonde/metrics/modules.rb

@@ -1,14 +1,19 @@
+# frozen_string_literal: true
+
+# modules plugin
 class Dropsonde::Metrics::Modules
   def self.initialize_modules
     # require any libraries needed here -- no need to load puppet; it's already initialized
     # All plugins are initialized before any metrics are generated.
+    require 'puppet/info_service'
+    require 'puppet/info_service/class_information_service'
   end
 
   def self.description
-    <<~EOF
+    <<~DESCRIPTION
       This group of metrics exports name & version information about the public
       modules installed in all environments, ignoring private modules.
-    EOF
+    DESCRIPTION
   end
 
   def self.schema
@@ -18,49 +23,89 @@ class Dropsonde::Metrics::Modules
       {
         "fields": [
           {
-            "description": "The module name",
-            "mode": "NULLABLE",
-            "name": "name",
-            "type": "STRING"
+            "description": 'The module name',
+            "mode": 'NULLABLE',
+            "name": 'name',
+            "type": 'STRING',
           },
           {
-            "description": "The module slug (author-name)",
-            "mode": "NULLABLE",
-            "name": "slug",
-            "type": "STRING"
+            "description": 'The module slug (author-name)',
+            "mode": 'NULLABLE',
+            "name": 'slug',
+            "type": 'STRING',
           },
           {
-            "description": "The module version",
-            "mode": "NULLABLE",
-            "name": "version",
-            "type": "STRING"
-          }
+            "description": 'The module version',
+            "mode": 'NULLABLE',
+            "name": 'version',
+            "type": 'STRING',
+          },
         ],
-        "description": "List of modules in all environments.",
-        "mode": "REPEATED",
-        "name": "modules",
-        "type": "RECORD"
+        "description": 'List of modules in all environments.',
+        "mode": 'REPEATED',
+        "name": 'modules',
+        "type": 'RECORD',
       },
       {
         "fields": [
           {
-            "description": "The class name",
-            "mode": "NULLABLE",
-            "name": "name",
-            "type": "STRING"
+            "description": 'The class name',
+            "mode": 'NULLABLE',
+            "name": 'name',
+            "type": 'STRING',
           },
           {
-            "description": "How many nodes it is declared on",
-            "mode": "NULLABLE",
-            "name": "count",
-            "type": "INTEGER"
-          }
+            "description": 'How many nodes it is declared on',
+            "mode": 'NULLABLE',
+            "name": 'count',
+            "type": 'INTEGER',
+          },
         ],
-        "description": "List of classes and counts in all environments.",
-        "mode": "REPEATED",
-        "name": "classes",
-        "type": "RECORD"
-      }
+        "description": 'List of classes and counts in all environments.',
+        "mode": 'REPEATED',
+        "name": 'classes',
+        "type": 'RECORD',
+      },
+      {
+        "fields": [
+          {
+            "description": 'The module name',
+            "mode": 'NULLABLE',
+            "name": 'name',
+            "type": 'STRING',
+          },
+          {
+            "description": 'The module slug (author-name)',
+            "mode": 'NULLABLE',
+            "name": 'slug',
+            "type": 'STRING',
+          },
+          {
+            "description": 'The module version',
+            "mode": 'NULLABLE',
+            "name": 'version',
+            "type": 'STRING',
+          },
+        ],
+        "description": 'List of modules whose classes are not declared in any environments.',
+        "mode": 'REPEATED',
+        "name": 'unused_modules',
+        "type": 'RECORD',
+      },
+      {
+        "fields": [
+          {
+            "description": 'The class name',
+            "mode": 'NULLABLE',
+            "name": 'name',
+            "type": 'STRING',
+          },
+        ],
+        "description": 'List of unused classes in all environments.',
+        "mode": 'REPEATED',
+        "name": 'unused_classes',
+        "type": 'RECORD',
+      },
     ]
   end
 
@@ -68,46 +113,76 @@ class Dropsonde::Metrics::Modules
     # run just before generating this metric
   end
 
-  def self.run
+  def self.run(puppetdb_session = nil)
     # return an array of hashes representing the data to be merged into the combined checkin
-    environments = Puppet.lookup(:environments).list.map{|e|e.name}
-    modules = environments.map do |env|
-      Puppet.lookup(:environments).get(env).modules.map do|mod|
+    environments = Puppet.lookup(:environments).list.map { |e| e.name }
+    modules = environments.map { |env|
+      Puppet.lookup(:environments).get(env).modules.map do |mod|
         next unless mod.forge_module?
 
         {
-          :name    => mod.name,
-          :slug    => mod.forge_slug,
-          :version => mod.version,
+          name: mod.name,
+          slug: mod.forge_slug,
+          version: mod.version,
         }
       end
-    end.flatten.compact.uniq
+    }.flatten.compact.uniq
 
-    if Dropsonde.puppetDB
+    if puppetdb_session
       # classes and how many nodes they're enforced on
-      results = Dropsonde.puppetDB.request( '',
-        'resources[type, title] { type = "Class" }'
-      ).data
+      results = puppetdb_session.puppet_db.request('', 'resources[type, title] { type = "Class" }').data
 
       # select only classes from public modules.
       # Use uniq to reduce the iteration over very large datasets
-      classes = results.uniq.map do |klass|
+      classes = results.uniq.map { |klass|
         title   = klass['title']
         modname = title.split('::').first.downcase
-        next unless modules.find {|mod| mod[:name] == modname }
+        next unless modules.find { |mod| mod[:name] == modname }
 
         {
-          :name  => title,
-          :count => results.count {|row| row['title'] == title},
+          name: title,
+          count: results.count { |row| row['title'] == title },
         }
-      end.compact
+      }.compact
+
+      # now lets get a list of all classes so we can identify which are unused
+      infoservice = Puppet::InfoService::ClassInformationService.new
+      env_hash = {}
+      environments.each do |env|
+        manifests = Puppet.lookup(:environments).get(env).modules.reduce([]) do |acc, mod|
+          next acc unless mod.forge_module?
+
+          acc.concat mod.all_manifests
+        end
+        env_hash[env] = manifests
+      end
+
+      klasses_per_env = infoservice.classes_per_environment(env_hash)
+
+      installed_classes = klasses_per_env.reduce([]) do |klasses, (_key, env)|
+        names = env.reduce([]) do |acc, (_file, contents)|
+          acc.concat(contents[:classes].map { |c| c[:name] })
+        end
+
+        klasses.concat names
+      end
+
+      unused_modules = installed_classes.map { |c| c.split('::').first }.sort.uniq
+      classes.each { |c| unused_modules.delete(c[:name].split('::').first.downcase) }
+
+      unused_classes = installed_classes.dup
+      classes.each { |c| unused_classes.delete(c[:name].downcase) }
     else
       classes = []
+      unused_modules  = []
+      unused_classes  = []
     end
 
     [
-      { :modules => modules },
-      { :classes => classes },
+      { modules: modules },
+      { classes: classes },
+      { unused_modules: unused_modules },
+      { unused_classes: unused_classes },
     ]
   end
 
@@ -119,19 +194,29 @@ class Dropsonde::Metrics::Modules
     versions = ['1.3.2', '0.0.1', '0.1.2', '1.0.0', '3.0.2', '7.10', '6.1.0', '2.1.0', '1.4.0']
     classes = ['', '::Config', '::Service', '::Server', '::Client', '::Packages']
     [
-      :modules => Dropsonde::Cache.modules
-                                  .sample(rand(100))
-                                  .map {|item| {
-                                    :name    => item.split('-').last,
-                                    :slug    => item,
-                                    :version => versions.sample,
-                                  }},
-      :classes => Dropsonde::Cache.modules
-                                  .sample(rand(500))
-                                  .map {|item| {
-                                    :name  => item.split('-').last.capitalize + classes.sample,
-                                    :count => rand(750),
-                                  }},
+      modules: Dropsonde::Cache.modules
+                               .sample(rand(100))
+                               .map do |item|
+                 {
+                   name: item.split('-').last,
+                   slug: item,
+                   version: versions.sample,
+                 }
+               end,
+      classes: Dropsonde::Cache.modules
+                               .sample(rand(500))
+                               .map do |item|
+                 {
+                   name: item.split('-').last.capitalize + classes.sample,
+                   count: rand(750),
+                 }
+               end,
+      unused_modules: Dropsonde::Cache.modules
+                                      .sample(rand(500))
+                                      .map { |item| item.split('-').last },
+      unused_classes: Dropsonde::Cache.modules
+                                      .sample(rand(500))
+                                      .map { |item| item.split('-').last.capitalize + classes.sample },
     ]
   end
 

data/lib/dropsonde/metrics/platforms.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+
+# platforms plugin
 class Dropsonde::Metrics::Platforms
   def self.initialize_platforms
     # require any libraries needed here -- no need to load puppet; it's already initialized
@@ -5,11 +8,11 @@ class Dropsonde::Metrics::Platforms
   end
 
   def self.description
-    <<~EOF
+    <<~DESCRIPTION
       This group of metrics generates usage patterns by platform.
       Currently implemented is a list of classes, the platforms
       they are declared on, and a count of each combination.
-    EOF
+    DESCRIPTION
   end
 
   def self.schema
@@ -19,29 +22,29 @@ class Dropsonde::Metrics::Platforms
       {
         "fields": [
           {
-            "description": "The class name name",
-            "mode": "NULLABLE",
-            "name": "name",
-            "type": "STRING"
+            "description": 'The class name name',
+            "mode": 'NULLABLE',
+            "name": 'name',
+            "type": 'STRING',
           },
           {
-            "description": "The osfamily of the node the class is declared on",
-            "mode": "NULLABLE",
-            "name": "platform",
-            "type": "STRING"
+            "description": 'The osfamily of the node the class is declared on',
+            "mode": 'NULLABLE',
+            "name": 'platform',
+            "type": 'STRING',
           },
           {
-            "description": "The number of time this combination is declared",
-            "mode": "NULLABLE",
-            "name": "count",
-            "type": "INTEGER"
+            "description": 'The number of time this combination is declared',
+            "mode": 'NULLABLE',
+            "name": 'count',
+            "type": 'INTEGER',
           },
         ],
         "description": "List of all classes in the infrastructure and platforms they're declared on.",
-        "mode": "REPEATED",
-        "name": "class_platforms",
-        "type": "RECORD"
-      }
+        "mode": 'REPEATED',
+        "name": 'class_platforms',
+        "type": 'RECORD',
+      },
     ]
   end
 
@@ -49,44 +52,43 @@ class Dropsonde::Metrics::Platforms
     # run just before generating this metric
   end
 
-  def self.run
+  def self.run(puppetdb_session = nil)
     # skip this metric if we don't have an active PuppetDB connection
-    return unless Dropsonde.puppetDB
+    return unless puppetdb_session
 
-    classes = Dropsonde.puppetDB.request( '', 'resources[certname, title] { type = "Class" }').data
-    facts   = Dropsonde.puppetDB.request( '', 'facts[certname, value] { name = "osfamily" }').data
+    classes = puppetdb_session.puppet_db.request('', 'resources[certname, title] { type = "Class" }').data
+    facts   = puppetdb_session.puppet_db.request('', 'facts[certname, value] { name = "osfamily" }').data
 
     # All public Forge modules that are installed.
-    modules = Puppet.lookup(:environments).list.map {|env|
-        env.modules.select {|mod|
-            mod.forge_module?
-        }.map {|fmod|
-            fmod.name
-        }}.flatten.uniq
-
-    data = classes.map {|item|
-        # filter out any that don't come from public Forge modules
-        mod = item['title'].split('::').first.downcase
-        next unless modules.include? mod
-
-        item['platform'] = facts.find {|fact|
-            fact['certname'] == item['certname']
-        }['value']
+    modules = Puppet.lookup(:environments).list.map { |env|
+      env.modules.select { |mod| mod.forge_module? }.map do |fmod|
+        fmod.name
+      end
+    }.flatten.uniq
 
-        {
-            :name     => item['title'],
-            :platform => item['platform'],
-        }
+    data = classes.map { |item|
+      # filter out any that don't come from public Forge modules
+      mod = item['title'].split('::').first.downcase
+      next unless modules.include? mod
+
+      item['platform'] = facts.find { |fact|
+        fact['certname'] == item['certname']
+      }['value']
+
+      {
+        name: item['title'],
+        platform: item['platform'],
+      }
     }.compact
 
-    data.each {|item|
-        item['count'] = data.select {|i|
-            i[:name] == item[:name] and i[:platform] == item[:platform]
-        }.count
-    }
+    data.each do |item|
+      item[:count] = data.select { |i|
+        i[:name] == item[:name] and i[:platform] == item[:platform]
+      }.count
+    end
 
     [
-      :class_platforms => data,
+      class_platforms: data,
     ]
   end
 
@@ -95,25 +97,25 @@ class Dropsonde::Metrics::Platforms
     # make it easier to write data aggregation queries without access to the
     # actual private data that users have submitted.
 
-    platforms = ['RedHat', 'Debian', 'Windows', 'Suse', 'FreeBSD', 'Darwin', 'Archlinux', 'AIX']
+    platforms = %w[RedHat Debian Windows Suse FreeBSD Darwin Archlinux AIX]
     classes   = ['', '::Config', '::Service', '::Server', '::Client', '::Packages']
 
     data = Dropsonde::Cache.modules
-            .sample(rand(35))
-            .map do |item|
-                name = item.split('-').last.capitalize + classes.sample
-
-                rand(5).times.map do
-                    {
-                        :name     => name,
-                        :platform => platforms.sample,
-                        :count    => rand(1000),
-                    }
-                end
-            end.flatten
+                           .sample(rand(35))
+                           .map { |item|
+      name = item.split('-').last.capitalize + classes.sample
+
+      rand(5).times.map do
+        {
+          name: name,
+          platform: platforms.sample,
+          count: rand(1000),
+        }
+      end
+    }.flatten
 
     [
-      :class_platforms => data.uniq,
+      class_platforms: data.uniq,
     ]
   end
 

data/lib/dropsonde/metrics/puppetfiles.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+
+# puppetfiles plugin
 class Dropsonde::Metrics::Puppetfiles
   def self.initialize_puppetfiles
     # require any libraries needed here -- no need to load puppet; it's already initialized
@@ -6,10 +9,10 @@ class Dropsonde::Metrics::Puppetfiles
   end
 
   def self.description
-    <<~EOF
+    <<~DESCRIPTION
       This generates interesting stats about Puppetfiles used in your environments,
       including whether your Puppetfiles have Ruby code in them.
-    EOF
+    DESCRIPTION
   end
 
   def self.schema
@@ -19,23 +22,23 @@ class Dropsonde::Metrics::Puppetfiles
       {
         "fields": [
           {
-            "description": "The method name",
-            "mode": "NULLABLE",
-            "name": "name",
-            "type": "STRING"
+            "description": 'The method name',
+            "mode": 'NULLABLE',
+            "name": 'name',
+            "type": 'STRING',
           },
           {
-            "description": "How many times is it used",
-            "mode": "NULLABLE",
-            "name": "count",
-            "type": "INTEGER"
-          }
+            "description": 'How many times is it used',
+            "mode": 'NULLABLE',
+            "name": 'count',
+            "type": 'INTEGER',
+          },
         ],
-        "description": "Ruby methods used in Puppetfiles.",
-        "mode": "REPEATED",
-        "name": "puppetfile_ruby_methods",
-        "type": "RECORD"
-      }
+        "description": 'Ruby methods used in Puppetfiles.',
+        "mode": 'REPEATED',
+        "name": 'puppetfile_ruby_methods',
+        "type": 'RECORD',
+      },
     ]
   end
 
@@ -43,30 +46,30 @@ class Dropsonde::Metrics::Puppetfiles
     # run just before generating this metric
   end
 
-  def self.run
-    methods = Dir.entries(Puppet.settings[:environmentpath]).map do |entry|
+  def self.run(_puppetdb_session = nil)
+    methods = Dir.entries(Puppet.settings[:environmentpath]).map { |entry|
       puppetfile = File.join(Puppet.settings[:environmentpath], entry, 'Puppetfile')
 
       next if entry.start_with? '.'
       next unless File.file? puppetfile
 
       tokens  = Ripper.sexp(File.read(puppetfile)).flatten
-      indices = tokens.map.with_index {|a, i| a == :command ? i : nil}.compact
+      indices = tokens.map.with_index { |a, i| (a == :command) ? i : nil }.compact
 
-      indices.map {|i| tokens[i+2] }
-    end.flatten.compact
+      indices.map { |i| tokens[i + 2] }
+    }.flatten.compact
 
-    methods.reject! {|name| ['mod', 'forge', 'moduledir'].include? name }
+    methods.reject! { |name| %w[mod forge moduledir].include? name }
 
     methods = methods.uniq.map do |name|
       {
-        :name  => name,
-        :count => methods.count(name),
+        name: name,
+        count: methods.count(name),
       }
     end
 
     [
-      { :puppetfile_ruby_methods => methods },
+      { puppetfile_ruby_methods: methods },
     ]
   end
 
@@ -75,13 +78,13 @@ class Dropsonde::Metrics::Puppetfiles
     # make it easier to write data aggregation queries without access to the
     # actual private data that users have submitted.
     [
-      :puppetfile_ruby_methods => [
-        {:name => 'require', :count => rand(200)},
-        {:name => 'each',    :count => rand(200)},
-        {:name => 'puts',    :count => rand(200)},
-        {:name => 'select',  :count => rand(200)},
-        {:name => 'reject',  :count => rand(200)},
-        {:name => 'read',    :count => rand(200)},
+      puppetfile_ruby_methods: [
+        { name: 'require', count: rand(200) },
+        { name: 'each',    count: rand(200) },
+        { name: 'puts',    count: rand(200) },
+        { name: 'select',  count: rand(200) },
+        { name: 'reject',  count: rand(200) },
+        { name: 'read',    count: rand(200) },
       ].shuffle,
     ]
   end