dropsonde 0.0.2 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4e4919cf3d11feb2d8a1a546bbc8ffd4cb4ebe182fcfe056aa39e7d382c88a9b
-  data.tar.gz: c0bee1c43b2e3b9d317fd79ca3f3dd568b8893ff99378c14e0f6ed53a7b934ef
+  metadata.gz: f4a87b2c8abe75d07a73a52a48041c2734bb7914a54df3471d696458f3f0cb05
+  data.tar.gz: cd2b7d3a36aa9aa386997ab21f7ffee57387819b08a79407b98c5bfdbbcd619d
 SHA512:
-  metadata.gz: b49fc015d8cecde93318d475a25e48c4eba104e454b3d701ba995d381df865907d10f44fc842ab2ce6dca0678a62eb15c3452453b55fe2292689e790fba30eb0
-  data.tar.gz: 337be044607d02a10ec7cd84f31560aab72d6c0a78f15f14292a9c1e528d1380c4a73be48a1677943d30bee728c9b88e6672158c123d2118941bab8af1efcd69
+  metadata.gz: 4a033ee5a796d3dab8c46d7562848255ca66552671f31f58ffde8a5a00f98dbcf1e9485a8279529b7c78094f96add8e932fa401e634ccfc067f5c9b72eed93c3
+  data.tar.gz: b86bc1f854fe414a388534ced35132d0744ac21001298d3eefce326afa97a63bd56ce9b250458889690831e7b0343963666377eaf10c25c38861b50e1160225d

data/README.md

@@ -43,10 +43,10 @@ schema so the system cannot gather any data that's not described in the schema.
 See the full schema of all enabled plugins by running the command:
 
 ```
-$ dropsonde schema
+$ dropsonde dev schema
 ```
 
-All information in the report is keyed off a non-reversible SHA512 hash site-id
+All information in the report is keyed off a non-reversible SHA512 hashed site-id
 to make it unidentifiable; this report cannot be linked back to you or to your
 infrastructure. Now that said, we know that the more bits of data shared about a
 specific site, the easier it is to fingerprint that site. See
@@ -63,7 +63,9 @@ For example, this aggregated data might include records that show a count of how
 many sites are using various combinations of modules together, but it will never
 include a record showing the full list of modules that any single site is using.
 
-With your own Google Cloud account, you can use that [dataset](https://console.cloud.google.com/bigquery?project=puppetlabs.com:api-project-53122606061)
+![Data aggregation workflow](./aggregation.png)
+
+With your own Google Cloud account, you can use that [dataset](https://console.cloud.google.com/bigquery?p=dataops-puppet-public-data&d=community&t=forge_modules&page=table)
 in your own tooling and you can see/contribute to the aggregation queries in its
 own [repository](https://github.com/puppetlabs/dropsonde-aggregation).
 
@@ -85,7 +87,9 @@ possible: [privacy@puppet.com](mailto:privacy@puppet.com)
 
 ## Installation
 
-This is distributed as a Ruby gem. Simply `gem install dropsonde`
+This is distributed as a Ruby gem. Simply `gem install dropsonde`. There's a
+[Puppet module](https://github.com/puppetlabs/puppetlabs-dropsonde) to manage it
+if that's more your thing.
 
 
 ## Configuration
@@ -98,7 +102,7 @@ will not report the `:puppetfiles` metrics.
 ``` yaml
 ---
 :update: false
-:blacklist:
+:disable:
   - puppetfiles
 ```
 
@@ -112,8 +116,6 @@ Run `dropsonde --help` to see usage information.
 * `preview`
     * Generate and print out an example telemetry report in human readable form
     * Annotated with descriptions of each plugin and each metric gathered.
-* `schema`
-    * Generate and print out the complete combined schema.
 * `list`
     * See a quick list of the available metrics and what they do.
 * `submit`
@@ -123,6 +125,17 @@ Run `dropsonde --help` to see usage information.
     * Once a week, the list of public modules on the Forge will be updated. This
       command will manually force that cache update to happen.
 
+Developer comands
+
+* `dev example`
+    * To make writing aggregation queries possible without access to the private
+      database, this will generate a randomized example of the dataset. This is
+      in JSONL format, so it can be imported directly into BigQuery.
+* `dev schema`
+    * Generate and print out the complete combined schema of all metrics.
+* `dev shell`
+    * Open up a Pry shell with all the relevant connections open and initialized.
+
 
 ## Architecture
 

data/bin/dropsonde

@@ -1,15 +1,15 @@
 #!/usr/bin/env ruby
 require 'gli'
 require 'dropsonde'
-require 'puppet'
 
 class Dropsonde
   extend GLI::App
 
-  Puppet.initialize_settings
+  @cache = nil
+  @puppetdb_session = Dropsonde.new
 
   program_desc 'A simple telemetry tool for Puppet infrastructures'
-  config_file  '/etc/puppetlabs/telemetry.yaml'
+  config_file  "#{File.dirname(Puppet.settings[:confdir])}/telemetry.yaml"
   version      Dropsonde::VERSION
 
   desc 'Verbose logging'
@@ -25,27 +25,26 @@ class Dropsonde
   flag [:ttl], :default_value => 7, :type => Integer
 
   desc 'List of metrics to omit'
-  flag [:blacklist, :b], :type => Array
+  flag [:disable, :d], :type => Array
+
+  desc 'Only load these metrics'
+  flag [:enable, :e], :type => Array
 
   desc 'Any number or string used to generate the randomized site ID.'
   flag [:seed]
 
+  desc 'Static site ID'
+  flag [:siteid]
+
   pre do |global, command, options, args|
     Dropsonde.settings = global
-    Dropsonde::Cache.init(global[:cachepath], global[:ttl], global[:update])
+    @cache = Dropsonde::Cache.new(global[:cachepath], global[:ttl], global[:update])
   end
 
   desc 'Manually update the Forge module name cache'
   command :update do |c|
     c.action do |global, options, args|
-      Dropsonde::Cache.update
-    end
-  end
-
-  desc 'Generate a complete schema set'
-  command :schema do |c|
-    c.action do |global, options, args|
-      Dropsonde.generate_schema
+      @cache.update
     end
   end
 
@@ -56,14 +55,14 @@ class Dropsonde
     end
   end
 
-  desc 'Generate an example telemetry report'
+  desc 'Preview the telemetry report that will be submitted'
   command :preview do |c|
     c.desc 'The output format to use'
     c.flag [:format], :default_value => 'human'
 
     c.action do |global, options, args|
-      Dropsonde::Cache.autoupdate
-      Dropsonde.generate_report(options[:format])
+      @cache.autoupdate
+      Dropsonde.generate_report(options[:format], @puppetdb_session)
     end
   end
 
@@ -76,10 +75,54 @@ class Dropsonde
     c.flag [:port], :default_value => 443, :type => Integer
 
     c.action do |global, options, args|
-      Dropsonde::Cache.autoupdate
+      @cache.autoupdate
       Dropsonde.submit_report(options[:endpoint], options[:port])
     end
   end
+
+  desc "Commands useful for developers"
+  command :dev do |t|
+    t.desc 'Open a Pry shell for debugging'
+    t.command :shell do |c|
+      c.action do |global, options, args|
+        require 'pry'
+        binding.pry
+      end
+    end
+
+    t.desc 'Generate a complete schema for all metrics'
+    t.long_desc "This generates the schema that is used to create or update the BigQuery
+                 database. Every report is also validated against this schema before
+                 submission, so you can be assured that this is a complete representation
+                 of what data is collected and run through aggregation filters."
+    t.command :schema do |c|
+      c.action do |global, options, args|
+        Dropsonde.generate_schema
+      end
+    end
+
+    t.desc 'Generate an example of random data to simulate actual reports'
+    t.long_desc "The submitted telemetry reports are treated as sensitive material. Very
+                few people have access to that raw data. Instead, it's run through some
+                data aggregation filters to generate the published statistics we share.
+                Writing those aggregation queries is difficult without data to work with,
+                so this command generates a representative example of random data.
+
+                This is in jsonl format for direct upload to BigQuery."
+    t.command :example do |c|
+      c.desc 'How many rows to generate'
+      c.flag [:size], :default_value => 100, :type => Integer
+
+      c.desc 'Filename for the output (in jsonl format).'
+      c.flag [:filename], :default_value => 'example.jsonl'
+
+      c.action do |global, options, args|
+        @cache.autoupdate
+        Dropsonde.generate_example(options[:size], options[:filename])
+      end
+    end
+  end
+
 end
 
 exit Dropsonde.run(ARGV)

data/lib/dropsonde/cache.rb

@@ -1,34 +1,41 @@
+# frozen_string_literal: true
+
 require 'date'
 require 'json'
 require 'fileutils'
 require 'puppet_forge'
 
+# cache class
 class Dropsonde::Cache
-  @@autoupdate = false
+  @autoupdate = false
 
-  def self.init(path, ttl, autoupdate)
+  def initialize(path, ttl, autoupdate)
     FileUtils.mkdir_p(path)
-    @@path = "#{File.expand_path(path)}/forge.json"
-    @@ttl  = ttl
-    @@autoupdate = autoupdate
-
-    if File.file? @@path
-      @@cache = JSON.parse(File.read(@@path))
-    else
-      @@cache = {
-                'timestamp' => '2000-1-1',  # long before any puppet modules were released!
-                'modules'   => [],
-              }
-    end
-
-    PuppetForge.user_agent = "Dropsonde Telemetry Client/0.0.1"
+    @path = "#{File.expand_path(path)}/forge.json"
+    @ttl  = ttl
+    @autoupdate = autoupdate
+
+    @@cache = if File.file? @path # rubocop:disable Style/ClassVars
+                JSON.parse(File.read(@path))
+              else
+                {
+                  'timestamp' => '2000-1-1', # long before any puppet modules were released!
+                  'modules' => [],
+                }
+              end
+
+    PuppetForge.user_agent = 'Dropsonde Telemetry Client/0.0.1'
   end
 
-  def self.modules
+  def modules
     @@cache['modules']
   end
 
-  def self.forgeModule?(mod)
+  def cache
+    @@cache
+  end
+
+  def self.forge_module?(mod)
     case mod
     when Puppet::Module
       modname = mod.forge_slug
@@ -39,38 +46,42 @@ class Dropsonde::Cache
     end
     return unless modname
 
-    modules.include? modname.tr('/','-')
+    @@cache['modules'].include? modname.tr('/', '-')
   end
 
-  def self.update
-    iter   = PuppetForge::Module.all(:sort_by => 'latest_release')
+  def update
+    puts 'Updating module cache...'
+    iter   = PuppetForge::Module.all(sort_by: 'latest_release')
     newest = DateTime.parse(@@cache['timestamp'])
 
-    @@cache['timestamp'] = iter.first.created_at
+    @@cache['timestamp'] = iter.first.updated_at
 
     until iter.next.nil?
       # stop once we reach modules we've already cached
-      break if DateTime.parse(iter.first.created_at) <= newest
+      break if DateTime.parse(iter.first.updated_at) <= newest
 
-      @@cache['modules'].concat iter.map {|mod| mod.slug }
+      @@cache['modules'].concat(iter.map { |mod| mod.slug })
 
       iter = iter.next
       print '.'
     end
+    puts
     @@cache['modules'].sort!
     @@cache['modules'].uniq!
 
-    File.write(@@path, JSON.pretty_generate(@@cache))
+    File.write(@path, JSON.pretty_generate(@@cache))
   end
 
-  def self.autoupdate
-    return unless @@autoupdate
-
-    update unless File.file? @@path
+  def autoupdate
+    return unless @autoupdate
 
-    if (Date.today - File.mtime(@@path).to_date).to_i > @@ttl
+    unless File.file? @path
+      puts 'Dropsonde caches a list of all Forge modules to ensure that it only reports'
+      puts 'usage data on public modules. Generating this cache may take some time on'
+      puts "the first run and you'll see your screen fill up with dots."
       update
     end
-  end
 
+    return update if (Date.today - File.mtime(@path).to_date).to_i > @ttl
+  end
 end

data/lib/dropsonde/metrics/dependencies.rb

@@ -1,3 +1,6 @@
+# frozen_string_literal: true
+
+# dependencies plugin
 class Dropsonde::Metrics::Dependencies
   def self.initialize_dependencies
     # require any libraries needed here -- no need to load puppet; it's already initialized
@@ -5,10 +8,10 @@ class Dropsonde::Metrics::Dependencies
   end
 
   def self.description
-    <<~EOF
+    <<~DESCRIPTION
       This group of metrics discovers dependencies between modules in all
       environments. It will omit dependencies on private modules.
-    EOF
+    DESCRIPTION
   end
 
   def self.schema
@@ -18,23 +21,23 @@ class Dropsonde::Metrics::Dependencies
       {
         "fields": [
           {
-            "description": "The depended on module name",
-            "mode": "NULLABLE",
-            "name": "name",
-            "type": "STRING"
+            "description": 'The depended on module name',
+            "mode": 'NULLABLE',
+            "name": 'name',
+            "type": 'STRING',
           },
           {
-            "description": "The depended on module version requirement",
-            "mode": "NULLABLE",
-            "name": "version_requirement",
-            "type": "STRING"
-          }
+            "description": 'The depended on module version requirement',
+            "mode": 'NULLABLE',
+            "name": 'version_requirement',
+            "type": 'STRING',
+          },
         ],
-        "description": "List of modules that private modules in all environments depend on.",
-        "mode": "REPEATED",
-        "name": "dependencies",
-        "type": "RECORD"
-      }
+        "description": 'List of modules that private modules in all environments depend on.',
+        "mode": 'REPEATED',
+        "name": 'dependencies',
+        "type": 'RECORD',
+      },
     ]
   end
 
@@ -42,26 +45,44 @@ class Dropsonde::Metrics::Dependencies
     # run just before generating this metric
   end
 
-  def self.run
+  def self.run(_puppetdb_session = nil)
     # return an array of hashes representing the data to be merged into the combined checkin
-    environments = Puppet.lookup(:environments).list.map{|e|e.name}
-    modules = environments.map do |env|
+    environments = Puppet.lookup(:environments).list.map { |e| e.name }
+    modules = environments.map { |env|
       Puppet.lookup(:environments).get(env).modules
-    end.flatten
+    }.flatten
 
     # we want only PUBLIC modules that PRIVATE modules depend on
-    dependencies = modules.map do|mod|
+    dependencies = modules.map { |mod|
       next unless mod.dependencies
-      next if Dropsonde::Cache.forgeModule? mod  # skip unless this is a private module
+      next if Dropsonde::Cache.forge_module? mod # skip unless this is a private module
 
       # and return a list of all public modules it depends on
-      mod.dependencies.select {|mod| Dropsonde::Cache.forgeModule? mod }
-    end.flatten.compact
+      mod.dependencies.select { |dep| Dropsonde::Cache.forge_module? dep }
+    }.flatten.compact
 
     [
-      { :dependencies => dependencies },
+      { dependencies: dependencies },
     ]
+  end
 
+  def self.example
+    # this method is used to generate a table filled with randomized data to
+    # make it easier to write data aggregation queries without access to the
+    # actual private data that users have submitted.
+
+    dropsonde_cache = Dropsonde::Cache.new('foo', 7, true)
+    versions = ['>= 1.5.2', '>= 4.3.2', '>= 3.0.0 < 4.0.0', '>= 2.2.1 < 5.0.0', '>= 5.0.0 < 7.0.0', '>= 4.11.0']
+    [
+      dependencies: dropsonde_cache.modules
+                                   .sample(rand(250))
+                                   .map do |item|
+                      {
+                        name: item,
+                        version_requirement: versions.sample,
+                      }
+                    end,
+    ]
   end
 
   def self.cleanup

data/lib/dropsonde/metrics/environments.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+# environments plugin
+class Dropsonde::Metrics::Environments
+  def self.initialize_environments
+    # Require any libraries needed here -- no need to load puppet or puppetdb;
+    # they're already loaded. This hook is named after the class name.
+    # All plugins are initialized at startup before any metrics are generated.
+  end
+
+  def self.description
+    # This is a Ruby squiggle heredoc; just a multi-line string with indentation removed
+    <<~DESCRIPTION
+      This group of metrics gathers information about environments.
+    DESCRIPTION
+  end
+
+  def self.schema
+    # return an array of hashes of a partial schema to be merged into the complete schema
+    [
+      {
+        "description": 'The number of environments',
+        "mode": 'NULLABLE',
+        "name": 'environment_count',
+        "type": 'INTEGER',
+      },
+    ]
+  end
+
+  def self.setup
+    # run just before generating this metric
+  end
+
+  def self.run(_puppetdb_session = nil)
+    # return an array of hashes representing the data to be merged into the combined checkin
+    [
+      environment_count: Puppet.lookup(:environments).list.count,
+    ]
+  end
+
+  def self.example
+    # this method is used to generate a table filled with randomized data to
+    # make it easier to write data aggregation queries without access to the
+    # actual private data that users have submitted.
+    [
+      environment_count: rand(1..100),
+    ]
+  end
+
+  def self.cleanup
+    # run just after generating this metric
+  end
+end