dropsonde 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a0fd6246263a8a375056fe6fdcb3bcc6b1db8cc6bddc7ea80f9dc870b48c85bc
-  data.tar.gz: b19a11ca730df12f61e95ae1f7bb81b9bbb65d8251966f3bcdd1fb203dacb96d
+  metadata.gz: 643def01638761803e04cf758bbf627ac821bb091fc7dad50c5b161251534728
+  data.tar.gz: f0c8a36548214da7beeecee9dc583dcd8d81eceb640b967078fadf5b2978f987
 SHA512:
-  metadata.gz: 53c7d74ceb0aa6358ddb12828faf70b840a33a4bbd30977317023050833a5db24d2b72eefc1eecfc1008e6e8eae80249b93a710edc098045c333787e4faa0ef2
-  data.tar.gz: 6644219dfdcbf761cb2c85bf446266ddf38d343d7a84d969ae88ffb30068c5faa647393457e1e255cd5fcbc2b8acb58f52d9efcd539c5713541932df79f0a807
+  metadata.gz: 7c260e1617666fb83d7b0be1a3d73ad13aaee5c33844d5eb4117b186c6eec23a9aad6dc03ff3df393168b6a3715bf492ce677624fdbe536e4961d956d1e9cdcb
+  data.tar.gz: cffbe7901afe08afa593ee73fa749358257efcd03d612564b3aac507684897d2d0db3e504a599d1eab7251508c3f455e236985501ea9f6722f108bb02cf80e2e

data/README.md

@@ -43,10 +43,10 @@ schema so the system cannot gather any data that's not described in the schema.
 See the full schema of all enabled plugins by running the command:
 
 ```
-$ dropsonde schema
+$ dropsonde dev schema
 ```
 
-All information in the report is keyed off a non-reversible SHA512 hash site-id
+All information in the report is keyed off a non-reversible SHA512 hashed site-id
 to make it unidentifiable; this report cannot be linked back to you or to your
 infrastructure. Now that said, we know that the more bits of data shared about a
 specific site, the easier it is to fingerprint that site. See
@@ -63,6 +63,8 @@ For example, this aggregated data might include records that show a count of how
 many sites are using various combinations of modules together, but it will never
 include a record showing the full list of modules that any single site is using.
 
+![Data aggregation workflow](./aggregation.png)
+
 With your own Google Cloud account, you can use that [dataset](https://console.cloud.google.com/bigquery?p=dataops-puppet-public-data&d=community&t=forge_modules&page=table)
 in your own tooling and you can see/contribute to the aggregation queries in its
 own [repository](https://github.com/puppetlabs/dropsonde-aggregation).
@@ -100,7 +102,7 @@ will not report the `:puppetfiles` metrics.
 ``` yaml
 ---
 :update: false
-:blacklist:
+:disable:
   - puppetfiles
 ```
 

data/bin/dropsonde

@@ -22,7 +22,10 @@ class Dropsonde
   flag [:ttl], :default_value => 7, :type => Integer
 
   desc 'List of metrics to omit'
-  flag [:blacklist, :b], :type => Array
+  flag [:disable, :d], :type => Array
+
+  desc 'Only load these metrics'
+  flag [:enable, :e], :type => Array
 
   desc 'Any number or string used to generate the randomized site ID.'
   flag [:seed]
@@ -102,7 +105,7 @@ class Dropsonde
                 This is in jsonl format for direct upload to BigQuery."
     t.command :example do |c|
       c.desc 'How many rows to generate'
-      c.flag [:size], :default_value => 100
+      c.flag [:size], :default_value => 100, :type => Integer
 
       c.desc 'Filename for the output (in jsonl format).'
       c.flag [:filename], :default_value => 'example.jsonl'

data/lib/dropsonde/metrics.rb

@@ -4,7 +4,13 @@ class Dropsonde::Metrics
   extend LittlePlugger( :path => 'dropsonde/metrics', :module => Dropsonde::Metrics)
 
   def initialize
-    Dropsonde::Metrics.disregard_plugins(*Dropsonde.settings[:blacklist])
+    if Dropsonde.settings[:enable]
+      Dropsonde.settings[:disable] ||= []
+      disable = Dropsonde::Metrics.plugins.keys - Dropsonde.settings[:enable].map(&:to_sym)
+      Dropsonde.settings[:disable].concat disable
+    end
+
+    Dropsonde::Metrics.disregard_plugins(*Dropsonde.settings[:disable])
     Dropsonde::Metrics.initialize_plugins
   end
 
@@ -28,9 +34,9 @@ class Dropsonde::Metrics
       str << plugin.description.strip
       str << "\n\n"
     end
-    if Dropsonde.settings[:blacklist]
+    if Dropsonde.settings[:disable]
       str << "Disabled plugins:\n"
-      str << "  #{Dropsonde.settings[:blacklist].join(', ')}"
+      str << "  #{Dropsonde.settings[:disable].join(', ')}"
     end
     str
   end
@@ -50,16 +56,16 @@ class Dropsonde::Metrics
     Dropsonde::Metrics.plugins.each do |name, plugin|
       schema = plugin.schema
 
-      plugin.setup
+      plugin.setup if plugin.respond_to? :setup
       data = sanity_check_data(plugin, plugin.run)
-      plugin.cleanup
+      plugin.cleanup if plugin.respond_to? :cleanup
 
       str << plugin.name+"\n"
       str << "-------------------------------\n"
       str << plugin.description
       data.each do |row|
         key    = row.keys.first
-        values = row.values.first
+        values = row.values.flatten
 
         desc = schema.find {|item| item[:name].to_sym == key.to_sym}[:description]
         str << "- #{key}: #{desc}\n"
@@ -112,6 +118,9 @@ class Dropsonde::Metrics
   # We accept both the plugin and data gathered from the plugin so that
   # we can sanitize both data and example data
   def sanity_check_data(plugin, data)
+    # This allows plugin authors to easily skip metrics with no results
+    return [] if data.nil?
+
     keys_data   = data.map {|item| item.keys }.flatten.map(&:to_s)
     keys_schema = plugin.schema.map {|item| item[:name] }
 

data/lib/dropsonde/metrics/environments.rb

@@ -0,0 +1,50 @@
+class Dropsonde::Metrics::Environments
+  def self.initialize_environments
+    # Require any libraries needed here -- no need to load puppet or puppetdb;
+    # they're already loaded. This hook is named after the class name.
+    # All plugins are initialized at startup before any metrics are generated.
+  end
+
+  def self.description
+    # This is a Ruby squiggle heredoc; just a multi-line string with indentation removed
+    <<~EOF
+      This group of metrics gathers information about environments.
+    EOF
+  end
+
+  def self.schema
+    # return an array of hashes of a partial schema to be merged into the complete schema
+    [
+      {
+        "description": "The number of environments",
+        "mode": "NULLABLE",
+        "name": "environment_count",
+        "type": "INTEGER"
+      }
+    ]
+  end
+
+  def self.setup
+    # run just before generating this metric
+  end
+
+  def self.run
+    # return an array of hashes representing the data to be merged into the combined checkin
+    [
+      :environment_count => Puppet.lookup(:environments).list.count,
+    ]
+  end
+
+  def self.example
+    # this method is used to generate a table filled with randomized data to
+    # make it easier to write data aggregation queries without access to the
+    # actual private data that users have submitted.
+    [
+      :environment_count => rand(1..100),
+    ]
+  end
+
+  def self.cleanup
+    # run just after generating this metric
+  end
+end

data/lib/dropsonde/metrics/platforms.rb

@@ -0,0 +1,123 @@
+class Dropsonde::Metrics::Platforms
+  def self.initialize_platforms
+    # require any libraries needed here -- no need to load puppet; it's already initialized
+    # All plugins are initialized before any metrics are generated.
+  end
+
+  def self.description
+    <<~EOF
+      This group of metrics generates usage patterns by platform.
+      Currently implemented is a list of classes, the platforms
+      they are declared on, and a count of each combination.
+    EOF
+  end
+
+  def self.schema
+    # return an array of hashes of a partial schema to be merged into the complete schema
+    # See https://cloud.google.com/bigquery/docs/schemas#specifying_a_json_schema_file
+    [
+      {
+        "fields": [
+          {
+            "description": "The class name name",
+            "mode": "NULLABLE",
+            "name": "name",
+            "type": "STRING"
+          },
+          {
+            "description": "The osfamily of the node the class is declared on",
+            "mode": "NULLABLE",
+            "name": "platform",
+            "type": "STRING"
+          },
+          {
+            "description": "The number of time this combination is declared",
+            "mode": "NULLABLE",
+            "name": "count",
+            "type": "INTEGER"
+          },
+        ],
+        "description": "List of all classes in the infrastructure and platforms they're declared on.",
+        "mode": "REPEATED",
+        "name": "class_platforms",
+        "type": "RECORD"
+      }
+    ]
+  end
+
+  def self.setup
+    # run just before generating this metric
+  end
+
+  def self.run
+    # skip this metric if we don't have an active PuppetDB connection
+    return unless Dropsonde.puppetDB
+
+    classes = Dropsonde.puppetDB.request( '', 'resources[certname, title] { type = "Class" }').data
+    facts   = Dropsonde.puppetDB.request( '', 'facts[certname, value] { name = "osfamily" }').data
+
+    # All public Forge modules that are installed.
+    modules = Puppet.lookup(:environments).list.map {|env|
+        env.modules.select {|mod|
+            mod.forge_module?
+        }.map {|fmod|
+            fmod.name
+        }}.flatten.uniq
+
+    data = classes.map {|item|
+        # filter out any that don't come from public Forge modules
+        mod = item['title'].split('::').first.downcase
+        next unless modules.include? mod
+
+        item['platform'] = facts.find {|fact|
+            fact['certname'] == item['certname']
+        }['value']
+
+        {
+            :name     => item['title'],
+            :platform => item['platform'],
+        }
+    }.compact
+
+    data.each {|item|
+        item['count'] = data.select {|i|
+            i[:name] == item[:name] and i[:platform] == item[:platform]
+        }.count
+    }
+
+    [
+      :class_platforms => data,
+    ]
+  end
+
+  def self.example
+    # this method is used to generate a table filled with randomized data to
+    # make it easier to write data aggregation queries without access to the
+    # actual private data that users have submitted.
+
+    platforms = ['RedHat', 'Debian', 'Windows', 'Suse', 'FreeBSD', 'Darwin', 'Archlinux', 'AIX']
+    classes   = ['', '::Config', '::Service', '::Server', '::Client', '::Packages']
+
+    data = Dropsonde::Cache.modules
+            .sample(rand(35))
+            .map do |item|
+                name = item.split('-').last.capitalize + classes.sample
+
+                rand(5).times.map do
+                    {
+                        :name     => name,
+                        :platform => platforms.sample,
+                        :count    => rand(1000),
+                    }
+                end
+            end.flatten
+
+    [
+      :class_platforms => data.uniq,
+    ]
+  end
+
+  def self.cleanup
+    # run just after generating this metric
+  end
+end

data/lib/dropsonde/version.rb

@@ -1,3 +1,3 @@
 class Dropsonde
-  VERSION = '0.0.4'
+  VERSION = '0.0.5'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dropsonde
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Ben Ford
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-26 00:00:00.000000000 Z
+date: 2020-06-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -153,7 +153,9 @@ files:
 - lib/dropsonde/cache.rb
 - lib/dropsonde/metrics.rb
 - lib/dropsonde/metrics/dependencies.rb
+- lib/dropsonde/metrics/environments.rb
 - lib/dropsonde/metrics/modules.rb
+- lib/dropsonde/metrics/platforms.rb
 - lib/dropsonde/metrics/puppetfiles.rb
 - lib/dropsonde/monkeypatches.rb
 - lib/dropsonde/version.rb