dropsonde 0.0.3 → 0.0.7

data/lib/dropsonde/metrics/modules.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+
+# modules plugin
 class Dropsonde::Metrics::Modules
   def self.initialize_modules
     # require any libraries needed here -- no need to load puppet; it's already initialized
@@ -5,10 +8,10 @@ class Dropsonde::Metrics::Modules
   end
 
   def self.description
-    <<~EOF
+    <<~DESCRIPTION
       This group of metrics exports name & version information about the public
       modules installed in all environments, ignoring private modules.
-    EOF
+    DESCRIPTION
   end
 
   def self.schema
@@ -18,49 +21,49 @@ class Dropsonde::Metrics::Modules
       {
         "fields": [
           {
-            "description": "The module name",
-            "mode": "NULLABLE",
-            "name": "name",
-            "type": "STRING"
+            "description": 'The module name',
+            "mode": 'NULLABLE',
+            "name": 'name',
+            "type": 'STRING',
           },
           {
-            "description": "The module slug (author-name)",
-            "mode": "NULLABLE",
-            "name": "slug",
-            "type": "STRING"
+            "description": 'The module slug (author-name)',
+            "mode": 'NULLABLE',
+            "name": 'slug',
+            "type": 'STRING',
           },
           {
-            "description": "The module version",
-            "mode": "NULLABLE",
-            "name": "version",
-            "type": "STRING"
-          }
+            "description": 'The module version',
+            "mode": 'NULLABLE',
+            "name": 'version',
+            "type": 'STRING',
+          },
         ],
-        "description": "List of modules in all environments.",
-        "mode": "REPEATED",
-        "name": "modules",
-        "type": "RECORD"
+        "description": 'List of modules in all environments.',
+        "mode": 'REPEATED',
+        "name": 'modules',
+        "type": 'RECORD',
       },
       {
         "fields": [
           {
-            "description": "The class name",
-            "mode": "NULLABLE",
-            "name": "name",
-            "type": "STRING"
+            "description": 'The class name',
+            "mode": 'NULLABLE',
+            "name": 'name',
+            "type": 'STRING',
           },
           {
-            "description": "How many nodes it is declared on",
-            "mode": "NULLABLE",
-            "name": "count",
-            "type": "INTEGER"
-          }
+            "description": 'How many nodes it is declared on',
+            "mode": 'NULLABLE',
+            "name": 'count',
+            "type": 'INTEGER',
+          },
         ],
-        "description": "List of classes and counts in all environments.",
-        "mode": "REPEATED",
-        "name": "classes",
-        "type": "RECORD"
-      }
+        "description": 'List of classes and counts in all environments.',
+        "mode": 'REPEATED',
+        "name": 'classes',
+        "type": 'RECORD',
+      },
     ]
   end
 
@@ -68,43 +71,44 @@ class Dropsonde::Metrics::Modules
     # run just before generating this metric
   end
 
-  def self.run
+  def self.run(puppetdb_session = nil)
     # return an array of hashes representing the data to be merged into the combined checkin
-    environments = Puppet.lookup(:environments).list.map{|e|e.name}
-    modules = environments.map do |env|
-      Puppet.lookup(:environments).get(env).modules.map do|mod|
+    environments = Puppet.lookup(:environments).list.map { |e| e.name }
+    modules = environments.map { |env|
+      Puppet.lookup(:environments).get(env).modules.map do |mod|
         next unless mod.forge_module?
 
         {
-          :name    => mod.name,
-          :slug    => mod.forge_slug,
-          :version => mod.version,
+          name: mod.name,
+          slug: mod.forge_slug,
+          version: mod.version,
         }
       end
-    end.flatten.compact.uniq
+    }.flatten.compact.uniq
 
-    if Dropsonde.puppetDB
+    if puppetdb_session
       # classes and how many nodes they're enforced on
-      results = Dropsonde.puppetDB.request( '',
-        'resources[certname, type, title] { type = "Class" }'
-      ).data
+      results = puppetdb_session.puppet_db.request('', 'resources[type, title] { type = "Class" }').data
 
-      # select only classes from public modules
-      classes = results.map do |klass|
-        next unless modules.find {|mod| mod[:name] == klass['title'].split('::').first.downcase }
+      # select only classes from public modules.
+      # Use uniq to reduce the iteration over very large datasets
+      classes = results.uniq.map { |klass|
+        title   = klass['title']
+        modname = title.split('::').first.downcase
+        next unless modules.find { |mod| mod[:name] == modname }
 
         {
-          :name  => klass['title'],
-          :count => results.count {|row| row['title'] == klass['title']},
+          name: title,
+          count: results.count { |row| row['title'] == title },
         }
-      end.compact.uniq
+      }.compact
     else
       classes = []
     end
 
     [
-      { :modules => modules },
-      { :classes => classes },
+      { modules: modules },
+      { classes: classes },
     ]
   end
 
@@ -115,20 +119,25 @@ class Dropsonde::Metrics::Modules
 
     versions = ['1.3.2', '0.0.1', '0.1.2', '1.0.0', '3.0.2', '7.10', '6.1.0', '2.1.0', '1.4.0']
     classes = ['', '::Config', '::Service', '::Server', '::Client', '::Packages']
+    dropsonde_cache = Dropsonde::Cache.new('foo', 7, true)
     [
-      :modules => Dropsonde::Cache.modules
-                                  .sample(rand(100))
-                                  .map {|item| {
-                                    :name    => item.split('-').last,
-                                    :slug    => item,
-                                    :version => versions.sample,
-                                  }},
-      :classes => Dropsonde::Cache.modules
-                                  .sample(rand(500))
-                                  .map {|item| {
-                                    :name  => item.split('-').last.capitalize + classes.sample,
-                                    :count => rand(750),
-                                  }},
+      modules: dropsonde_cache.modules
+                              .sample(rand(100))
+                              .map do |item|
+                 {
+                   name: item.split('-').last,
+                   slug: item,
+                   version: versions.sample,
+                 }
+               end,
+      classes: dropsonde_cache.modules
+                              .sample(rand(500))
+                              .map do |item|
+                 {
+                   name: item.split('-').last.capitalize + classes.sample,
+                   count: rand(750),
+                 }
+               end,
     ]
   end
 

data/lib/dropsonde/metrics/platforms.rb

@@ -0,0 +1,126 @@
+# frozen_string_literal: true
+
+# platforms plugin
+class Dropsonde::Metrics::Platforms
+  def self.initialize_platforms
+    # require any libraries needed here -- no need to load puppet; it's already initialized
+    # All plugins are initialized before any metrics are generated.
+  end
+
+  def self.description
+    <<~DESCRIPTION
+      This group of metrics generates usage patterns by platform.
+      Currently implemented is a list of classes, the platforms
+      they are declared on, and a count of each combination.
+    DESCRIPTION
+  end
+
+  def self.schema
+    # return an array of hashes of a partial schema to be merged into the complete schema
+    # See https://cloud.google.com/bigquery/docs/schemas#specifying_a_json_schema_file
+    [
+      {
+        "fields": [
+          {
+            "description": 'The class name name',
+            "mode": 'NULLABLE',
+            "name": 'name',
+            "type": 'STRING',
+          },
+          {
+            "description": 'The osfamily of the node the class is declared on',
+            "mode": 'NULLABLE',
+            "name": 'platform',
+            "type": 'STRING',
+          },
+          {
+            "description": 'The number of time this combination is declared',
+            "mode": 'NULLABLE',
+            "name": 'count',
+            "type": 'INTEGER',
+          },
+        ],
+        "description": "List of all classes in the infrastructure and platforms they're declared on.",
+        "mode": 'REPEATED',
+        "name": 'class_platforms',
+        "type": 'RECORD',
+      },
+    ]
+  end
+
+  def self.setup
+    # run just before generating this metric
+  end
+
+  def self.run(puppetdb_session = nil)
+    # skip this metric if we don't have an active PuppetDB connection
+    return unless puppetdb_session
+
+    classes = puppetdb_session.puppet_db.request('', 'resources[certname, title] { type = "Class" }').data
+    facts   = puppetdb_session.puppet_db.request('', 'facts[certname, value] { name = "osfamily" }').data
+
+    # All public Forge modules that are installed.
+    modules = Puppet.lookup(:environments).list.map { |env|
+      env.modules.select { |mod| mod.forge_module? }.map do |fmod|
+        fmod.name
+      end
+    }.flatten.uniq
+
+    data = classes.map { |item|
+      # filter out any that don't come from public Forge modules
+      mod = item['title'].split('::').first.downcase
+      next unless modules.include? mod
+
+      item['platform'] = facts.find { |fact|
+        fact['certname'] == item['certname']
+      }['value']
+
+      {
+        name: item['title'],
+        platform: item['platform'],
+      }
+    }.compact
+
+    data.each do |item|
+      item[:count] = data.select { |i|
+        i[:name] == item[:name] and i[:platform] == item[:platform]
+      }.count
+    end
+
+    [
+      class_platforms: data,
+    ]
+  end
+
+  def self.example
+    # this method is used to generate a table filled with randomized data to
+    # make it easier to write data aggregation queries without access to the
+    # actual private data that users have submitted.
+
+    platforms = %w[RedHat Debian Windows Suse FreeBSD Darwin Archlinux AIX]
+    classes   = ['', '::Config', '::Service', '::Server', '::Client', '::Packages']
+
+    dropsonde_cache = Dropsonde::Cache.new('foo', 7, true)
+    data = dropsonde_cache.modules
+                          .sample(rand(35))
+                          .map { |item|
+      name = item.split('-').last.capitalize + classes.sample
+
+      rand(5).times.map do
+        {
+          name: name,
+          platform: platforms.sample,
+          count: rand(1000),
+        }
+      end
+    }.flatten
+
+    [
+      class_platforms: data.uniq,
+    ]
+  end
+
+  def self.cleanup
+    # run just after generating this metric
+  end
+end

data/lib/dropsonde/metrics/puppetfiles.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+
+# puppetfiles plugin
 class Dropsonde::Metrics::Puppetfiles
   def self.initialize_puppetfiles
     # require any libraries needed here -- no need to load puppet; it's already initialized
@@ -6,10 +9,10 @@ class Dropsonde::Metrics::Puppetfiles
   end
 
   def self.description
-    <<~EOF
+    <<~DESCRIPTION
       This generates interesting stats about Puppetfiles used in your environments,
       including whether your Puppetfiles have Ruby code in them.
-    EOF
+    DESCRIPTION
   end
 
   def self.schema
@@ -19,23 +22,23 @@ class Dropsonde::Metrics::Puppetfiles
       {
         "fields": [
           {
-            "description": "The method name",
-            "mode": "NULLABLE",
-            "name": "name",
-            "type": "STRING"
+            "description": 'The method name',
+            "mode": 'NULLABLE',
+            "name": 'name',
+            "type": 'STRING',
           },
           {
-            "description": "How many times is it used",
-            "mode": "NULLABLE",
-            "name": "count",
-            "type": "INTEGER"
-          }
+            "description": 'How many times is it used',
+            "mode": 'NULLABLE',
+            "name": 'count',
+            "type": 'INTEGER',
+          },
         ],
-        "description": "Ruby methods used in Puppetfiles.",
-        "mode": "REPEATED",
-        "name": "puppetfile_ruby_methods",
-        "type": "RECORD"
-      }
+        "description": 'Ruby methods used in Puppetfiles.',
+        "mode": 'REPEATED',
+        "name": 'puppetfile_ruby_methods',
+        "type": 'RECORD',
+      },
     ]
   end
 
@@ -43,30 +46,30 @@ class Dropsonde::Metrics::Puppetfiles
     # run just before generating this metric
   end
 
-  def self.run
-    methods = Dir.entries(Puppet.settings[:environmentpath]).map do |entry|
+  def self.run(_puppetdb_session = nil)
+    methods = Dir.entries(Puppet.settings[:environmentpath]).map { |entry|
       puppetfile = File.join(Puppet.settings[:environmentpath], entry, 'Puppetfile')
 
       next if entry.start_with? '.'
       next unless File.file? puppetfile
 
       tokens  = Ripper.sexp(File.read(puppetfile)).flatten
-      indices = tokens.map.with_index {|a, i| a == :command ? i : nil}.compact
+      indices = tokens.map.with_index { |a, i| a == :command ? i : nil }.compact
 
-      indices.map {|i| tokens[i+2] }
-    end.flatten.compact
+      indices.map { |i| tokens[i + 2] }
+    }.flatten.compact
 
-    methods.reject! {|name| ['mod', 'forge', 'moduledir'].include? name }
+    methods.reject! { |name| %w[mod forge moduledir].include? name }
 
     methods = methods.uniq.map do |name|
       {
-        :name  => name,
-        :count => methods.count(name),
+        name: name,
+        count: methods.count(name),
       }
     end
 
     [
-      { :puppetfile_ruby_methods => methods },
+      { puppetfile_ruby_methods: methods },
     ]
   end
 
@@ -75,13 +78,13 @@ class Dropsonde::Metrics::Puppetfiles
     # make it easier to write data aggregation queries without access to the
     # actual private data that users have submitted.
     [
-      :puppetfile_ruby_methods => [
-        {:name => 'require', :count => rand(200)},
-        {:name => 'each',    :count => rand(200)},
-        {:name => 'puts',    :count => rand(200)},
-        {:name => 'select',  :count => rand(200)},
-        {:name => 'reject',  :count => rand(200)},
-        {:name => 'read',    :count => rand(200)},
+      puppetfile_ruby_methods: [
+        { name: 'require', count: rand(200) },
+        { name: 'each',    count: rand(200) },
+        { name: 'puts',    count: rand(200) },
+        { name: 'select',  count: rand(200) },
+        { name: 'reject',  count: rand(200) },
+        { name: 'read',    count: rand(200) },
       ].shuffle,
     ]
   end