dropsonde 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4e4919cf3d11feb2d8a1a546bbc8ffd4cb4ebe182fcfe056aa39e7d382c88a9b
-  data.tar.gz: c0bee1c43b2e3b9d317fd79ca3f3dd568b8893ff99378c14e0f6ed53a7b934ef
+  metadata.gz: 0ac6f28ed9d64982c400f6a7460b57f44396020e6f17f04f587cc6e5007ff115
+  data.tar.gz: ef5a2ac1957b4b45c756b0e4bbc5ae66291706afabd3a7b80e899702f8aa6757
 SHA512:
-  metadata.gz: b49fc015d8cecde93318d475a25e48c4eba104e454b3d701ba995d381df865907d10f44fc842ab2ce6dca0678a62eb15c3452453b55fe2292689e790fba30eb0
-  data.tar.gz: 337be044607d02a10ec7cd84f31560aab72d6c0a78f15f14292a9c1e528d1380c4a73be48a1677943d30bee728c9b88e6672158c123d2118941bab8af1efcd69
+  metadata.gz: d9bda687091ec5a1d6bd8ca7e5f1414e2c489e3a1c66a970b879d7f6a2d6ab82e11f079eebfea13ed7a710ebe20b4c0aefd0a08f1af17547e3a5cd120c587847
+  data.tar.gz: e96d699c38c0d158bce376f13d77d9bafa3ef5d4020e104d32694df04fcc68c91bb458d2bb858f9d120dc5b9146ffd3bfde450535cfe0b971b331b349e7dc78b

data/README.md

@@ -63,7 +63,7 @@ For example, this aggregated data might include records that show a count of how
 many sites are using various combinations of modules together, but it will never
 include a record showing the full list of modules that any single site is using.
 
-With your own Google Cloud account, you can use that [dataset](https://console.cloud.google.com/bigquery?project=puppetlabs.com:api-project-53122606061)
+With your own Google Cloud account, you can use that [dataset](https://console.cloud.google.com/bigquery?p=dataops-puppet-public-data&d=community&t=forge_modules&page=table)
 in your own tooling and you can see/contribute to the aggregation queries in its
 own [repository](https://github.com/puppetlabs/dropsonde-aggregation).
 
@@ -85,7 +85,9 @@ possible: [privacy@puppet.com](mailto:privacy@puppet.com)
 
 ## Installation
 
-This is distributed as a Ruby gem. Simply `gem install dropsonde`
+This is distributed as a Ruby gem. Simply `gem install dropsonde`. There's a
+[Puppet module](https://github.com/puppetlabs/puppetlabs-dropsonde) to manage it
+if that's more your thing.
 
 
 ## Configuration
@@ -112,8 +114,6 @@ Run `dropsonde --help` to see usage information.
 * `preview`
     * Generate and print out an example telemetry report in human readable form
     * Annotated with descriptions of each plugin and each metric gathered.
-* `schema`
-    * Generate and print out the complete combined schema.
 * `list`
     * See a quick list of the available metrics and what they do.
 * `submit`
@@ -123,6 +123,17 @@ Run `dropsonde --help` to see usage information.
     * Once a week, the list of public modules on the Forge will be updated. This
       command will manually force that cache update to happen.
 
+Developer comands
+
+* `dev example`
+    * To make writing aggregation queries possible without access to the private
+      database, this will generate a randomized example of the dataset. This is
+      in JSONL format, so it can be imported directly into BigQuery.
+* `dev schema`
+    * Generate and print out the complete combined schema of all metrics.
+* `dev shell`
+    * Open up a Pry shell with all the relevant connections open and initialized.
+
 
 ## Architecture
 

data/bin/dropsonde

@@ -1,15 +1,12 @@
 #!/usr/bin/env ruby
 require 'gli'
 require 'dropsonde'
-require 'puppet'
 
 class Dropsonde
   extend GLI::App
 
-  Puppet.initialize_settings
-
   program_desc 'A simple telemetry tool for Puppet infrastructures'
-  config_file  '/etc/puppetlabs/telemetry.yaml'
+  config_file  "#{File.dirname(Puppet.settings[:confdir])}/telemetry.yaml"
   version      Dropsonde::VERSION
 
   desc 'Verbose logging'
@@ -42,13 +39,6 @@ class Dropsonde
     end
   end
 
-  desc 'Generate a complete schema set'
-  command :schema do |c|
-    c.action do |global, options, args|
-      Dropsonde.generate_schema
-    end
-  end
-
   desc 'List all available metrics'
   command :list do |c|
     c.action do |global, options, args|
@@ -56,7 +46,7 @@ class Dropsonde
     end
   end
 
-  desc 'Generate an example telemetry report'
+  desc 'Preview the telemetry report that will be submitted'
   command :preview do |c|
     c.desc 'The output format to use'
     c.flag [:format], :default_value => 'human'
@@ -80,6 +70,50 @@ class Dropsonde
       Dropsonde.submit_report(options[:endpoint], options[:port])
     end
   end
+
+  desc "Commands useful for developers"
+  command :dev do |t|
+    t.desc 'Open a Pry shell for debugging'
+    t.command :shell do |c|
+      c.action do |global, options, args|
+        require 'pry'
+        binding.pry
+      end
+    end
+
+    t.desc 'Generate a complete schema for all metrics'
+    t.long_desc "This generates the schema that is used to create or update the BigQuery
+                 database. Every report is also validated against this schema before
+                 submission, so you can be assured that this is a complete representation
+                 of what data is collected and run through aggregation filters."
+    t.command :schema do |c|
+      c.action do |global, options, args|
+        Dropsonde.generate_schema
+      end
+    end
+
+    t.desc 'Generate an example of random data to simulate actual reports'
+    t.long_desc "The submitted telemetry reports are treated as sensitive material. Very
+                few people have access to that raw data. Instead, it's run through some
+                data aggregation filters to generate the published statistics we share.
+                Writing those aggregation queries is difficult without data to work with,
+                so this command generates a representative example of random data.
+
+                This is in jsonl format for direct upload to BigQuery."
+    t.command :example do |c|
+      c.desc 'How many rows to generate'
+      c.flag [:size], :default_value => 100
+
+      c.desc 'Filename for the output (in jsonl format).'
+      c.flag [:filename], :default_value => 'example.jsonl'
+
+      c.action do |global, options, args|
+        Dropsonde::Cache.autoupdate
+        Dropsonde.generate_example(options[:size], options[:filename])
+      end
+    end
+  end
+
 end
 
 exit Dropsonde.run(ARGV)

data/lib/dropsonde.rb

@@ -2,6 +2,7 @@ require 'json'
 require 'httpclient'
 require 'puppetdb'
 require 'inifile'
+require 'puppet'
 
 class Dropsonde
   require 'dropsonde/cache'
@@ -9,6 +10,8 @@ class Dropsonde
   require 'dropsonde/monkeypatches'
   require 'dropsonde/version'
 
+  Puppet.initialize_settings
+
   @@pdbclient = nil
   @@settings  = {}
   def self.settings=(arg)
@@ -63,6 +66,16 @@ class Dropsonde
     end
   end
 
+  def self.generate_example(size, filename)
+    metrics = Dropsonde::Metrics.new
+    File.open(filename, 'w') do |file|
+      for i in 0...size
+        file.write(metrics.example.to_json)
+        file.write("\n")
+      end
+    end
+  end
+
   def self.puppetDB
     return @@pdbclient if @@pdbclient
 

data/lib/dropsonde/cache.rb

@@ -43,20 +43,22 @@ class Dropsonde::Cache
   end
 
   def self.update
+    puts "Updating module cache..."
     iter   = PuppetForge::Module.all(:sort_by => 'latest_release')
     newest = DateTime.parse(@@cache['timestamp'])
 
-    @@cache['timestamp'] = iter.first.created_at
+    @@cache['timestamp'] = iter.first.updated_at
 
     until iter.next.nil?
       # stop once we reach modules we've already cached
-      break if DateTime.parse(iter.first.created_at) <= newest
+      break if DateTime.parse(iter.first.updated_at) <= newest
 
       @@cache['modules'].concat iter.map {|mod| mod.slug }
 
       iter = iter.next
       print '.'
     end
+    puts
     @@cache['modules'].sort!
     @@cache['modules'].uniq!
 
@@ -66,7 +68,12 @@ class Dropsonde::Cache
   def self.autoupdate
     return unless @@autoupdate
 
-    update unless File.file? @@path
+    unless File.file? @@path
+      puts "Dropsonde caches a list of all Forge modules to ensure that it only reports"
+      puts "usage data on public modules. Generating this cache may take some time on"
+      puts "the first run and you'll see your screen fill up with dots."
+      update
+    end
 
     if (Date.today - File.mtime(@@path).to_date).to_i > @@ttl
       update

data/lib/dropsonde/metrics.rb

@@ -78,7 +78,7 @@ class Dropsonde::Metrics
     snapshots = {}
     Dropsonde::Metrics.plugins.each do |name, plugin|
       plugin.setup
-      sanity_check_data(plugin).each do |row|
+      sanity_check_data(plugin, plugin.run).each do |row|
         snapshots[row.keys.first] = {
           'value'     => row.values.first,
           'timestamp' => Time.now.iso8601,
@@ -92,8 +92,26 @@ class Dropsonde::Metrics
     results
   end
 
-  def sanity_check_data(plugin)
-    data = plugin.run
+  def example
+    require 'ipaddr'
+    results = skeleton_report
+    results[:message_id] = generate_guid
+    results[:timestamp]  = rand((Time.now - 60 * 60 * 24 * 365)..Time.now).utc
+    results[:ip]         = IPAddr.new(rand(2**32), Socket::AF_INET)
+    results.delete(:'self-service-analytics')
+
+    Dropsonde::Metrics.plugins.each do |name, plugin|
+      sanity_check_data(plugin, plugin.example).each do |row|
+        results.merge!(row)
+      end
+    end
+
+    results
+  end
+
+  # We accept both the plugin and data gathered from the plugin so that
+  # we can sanitize both data and example data
+  def sanity_check_data(plugin, data)
     keys_data   = data.map {|item| item.keys }.flatten.map(&:to_s)
     keys_schema = plugin.schema.map {|item| item[:name] }
 
@@ -181,4 +199,14 @@ class Dropsonde::Metrics
       }
     }
   end
+
+  def generate_guid
+    "%s-%s-%s-%s-%s" % [
+      (0..8).to_a.map{|a| rand(16).to_s(16)}.join,
+      (0..4).to_a.map{|a| rand(16).to_s(16)}.join,
+      (0..4).to_a.map{|a| rand(16).to_s(16)}.join,
+      (0..4).to_a.map{|a| rand(16).to_s(16)}.join,
+      (0..12).to_a.map{|a| rand(16).to_s(16)}.join
+    ]
+  end
 end

data/lib/dropsonde/metrics/dependencies.rb

@@ -64,6 +64,22 @@ class Dropsonde::Metrics::Dependencies
 
   end
 
+  def self.example
+    # this method is used to generate a table filled with randomized data to
+    # make it easier to write data aggregation queries without access to the
+    # actual private data that users have submitted.
+
+    versions = ['>= 1.5.2', '>= 4.3.2', '>= 3.0.0 < 4.0.0', '>= 2.2.1 < 5.0.0', '>= 5.0.0 < 7.0.0', '>= 4.11.0']
+    [
+      :dependencies => Dropsonde::Cache.modules
+                                  .sample(rand(250))
+                                  .map {|item| {
+                                    :name                => item,
+                                    :version_requirement => versions.sample,
+                                  }},
+    ]
+  end
+
   def self.cleanup
     # run just after generating this metric
   end

data/lib/dropsonde/metrics/modules.rb

@@ -108,6 +108,30 @@ class Dropsonde::Metrics::Modules
     ]
   end
 
+  def self.example
+    # this method is used to generate a table filled with randomized data to
+    # make it easier to write data aggregation queries without access to the
+    # actual private data that users have submitted.
+
+    versions = ['1.3.2', '0.0.1', '0.1.2', '1.0.0', '3.0.2', '7.10', '6.1.0', '2.1.0', '1.4.0']
+    classes = ['', '::Config', '::Service', '::Server', '::Client', '::Packages']
+    [
+      :modules => Dropsonde::Cache.modules
+                                  .sample(rand(100))
+                                  .map {|item| {
+                                    :name    => item.split('-').last,
+                                    :slug    => item,
+                                    :version => versions.sample,
+                                  }},
+      :classes => Dropsonde::Cache.modules
+                                  .sample(rand(500))
+                                  .map {|item| {
+                                    :name  => item.split('-').last.capitalize + classes.sample,
+                                    :count => rand(750),
+                                  }},
+    ]
+  end
+
   def self.cleanup
     # run just after generating this metric
   end

data/lib/dropsonde/metrics/puppetfiles.rb

@@ -70,6 +70,22 @@ class Dropsonde::Metrics::Puppetfiles
     ]
   end
 
+  def self.example
+    # this method is used to generate a table filled with randomized data to
+    # make it easier to write data aggregation queries without access to the
+    # actual private data that users have submitted.
+    [
+      :puppetfile_ruby_methods => [
+        {:name => 'require', :count => rand(200)},
+        {:name => 'each',    :count => rand(200)},
+        {:name => 'puts',    :count => rand(200)},
+        {:name => 'select',  :count => rand(200)},
+        {:name => 'reject',  :count => rand(200)},
+        {:name => 'read',    :count => rand(200)},
+      ].shuffle,
+    ]
+  end
+
   def self.cleanup
     # run just after generating this metric
   end

data/lib/dropsonde/version.rb

@@ -1,3 +1,3 @@
 class Dropsonde
-  VERSION = '0.0.2'
+  VERSION = '0.0.3'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: dropsonde
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Ben Ford
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-05-07 00:00:00.000000000 Z
+date: 2020-05-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json