dropsonde 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 2972f1e6d2deff8b7826689d08c28f46b496f98af11ee98222f0cf9d67ea9d5e
+  data.tar.gz: ffc4b8c331ed8e18ee635f986877c88f590afb42d8bdbbb601d96ac9d37f6f65
+SHA512:
+  metadata.gz: 0e8a91a089a3cee0b96896216c21daa94b9715b06ab539b32956016ed8d563ca698e39245e4188f5b0d52f14bb05e46fe4fbb2484ceff80a23fa3e3742cf09b2
+  data.tar.gz: 7ad9668cc0f244f241ed7335f4f52b3745f437e73e41b60c302e9daa17354d7f75da8b35c8610d264361c7f95753cc65b0a4dc087f6af1244dabfc40c8a75004

data/LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -0,0 +1,152 @@
+# Dropsonde
+
+A simple telemetry probe for gathering non-identifiable information about Puppet
+infrastructures.
+
+
+## Overview
+
+We both know that you hate telemetry as much as I do. So what makes this different?
+At its core, the purpose and design of this module is for your own benefit as much
+as it is for ours. Think back to the time you last visited the Forge to find a
+module. Chances are that you discovered many modules that claimed to solve your
+problem and it was relatively difficult choosing between them. Surfacing usage
+data in a way that lets you signal approval simply by using a module is the
+primary goal of this project.
+
+> **This means that the best way for you to help yourself find new modules is to
+> install this telemetry tool and, along with your peers, share your module usage
+> data.** ✅
+
+
+## Design
+
+So what sort of information is gathered? Dropsonde identifies information like
+which public modules and classes are used, and what platforms they're used on.
+It identifies the component classes that your profile modules declares so we can
+better support what you are doing with your profiles, *but without exposing your
+profile names.* The key point here is that it looks for usage patterns for *public
+modules only* and explicitly does its best to keep your internal code private to you.
+
+You can see exactly what will be phoned home by running the command:
+
+```
+$ dropsonde preview
+```
+
+Dropsonde is a simple telemetry probe designed to run as a regular cron job.
+Metrics are defined by plugins that gather data, but also export a schema that
+constrains the data allowed to be reported on. Dropsonde will reject metrics
+that don't meet these constraints. The backend database is also defined by this
+schema so the system cannot gather any data that's not described in the schema.
+
+See the full schema of all enabled plugins by running the command:
+
+```
+$ dropsonde schema
+```
+
+All information in the report is keyed off a non-reversible SHA512 hash site-id
+to make it unidentifiable; this report cannot be linked back to you or to your
+infrastructure. Now that said, we know that the more bits of data shared about a
+specific site, the easier it is to fingerprint that site. See
+[Panopticlick](https://panopticlick.eff.org) for a real-world example.
+
+To mitigate this concern, we aggregate data in a two step process. The data
+submitted by Dropsonde is stored in a private dataset. ACLs limit access to only
+certain number of employees. Then each week, a job runs to generate a sanitized
+and aggregated form of the data for the week which goes into the public dataset
+that all our tooling actually depends on and uses. This dataset includes patterns
+and aggregate data, but does not include any individual records whatsoever.
+
+For example, this aggregated data might include records that show a count of how
+many sites are using various combinations of modules together, but it will never
+include a record showing the full list of modules that any single site is using.
+
+With your own Google Cloud account, you can use that [dataset](https://console.cloud.google.com/bigquery?project=puppetlabs.com:api-project-53122606061)
+in your own tooling and you can see/contribute to the aggregation queries in its
+own [repository](https://github.com/puppetlabs/dropsonde-aggregation).
+
+
+## Privacy
+
+Dropsonde will not collect the names or titles of internal modules, classes,
+facts, types, etc. It will gather the names of the component classes that your
+profiles use, but it will not gather the names of your profiles themselves. In
+order to maintain data integrity, reports are keyed off a non-reversible site-id.
+This means that there's no direct link from a record to you or your infrastructure,
+but it does mean that the data could be used for fingerprinting. To mitigate that,
+the data goes through an additional aggregation step that removes this ID and
+presents only usage patterns publicly.
+
+If you identify an unmitigated privacy concern then please inform us as soon as
+possible: [privacy@puppet.com](mailto:privacy@puppet.com)
+
+
+## Installation
+
+This is distributed as a Ruby gem. Simply `gem install dropsonde`
+
+
+## Configuration
+
+Any command line arguments can also be specified in `/etc/puppetlabs/telemetry.yaml`.
+For example the config file below will disable Forge module cache updating and
+will not report the `:puppetfiles` metrics.
+
+
+``` yaml
+---
+:update: false
+:blacklist:
+  - puppetfiles
+```
+
+The `puppetlabs-dropsonde` Puppet module manages this configuration for you.
+
+
+## Running
+
+Run `dropsonde --help` to see usage information.
+
+* `preview`
+    * Generate and print out an example telemetry report in human readable form
+    * Annotated with descriptions of each plugin and each metric gathered.
+* `schema`
+    * Generate and print out the complete combined schema.
+* `list`
+    * See a quick list of the available metrics and what they do.
+* `submit`
+    * Generate and submit a telemetry report. This will be exactly the same as
+      the human readable form, just in JSON format.
+* `update`
+    * Once a week, the list of public modules on the Forge will be updated. This
+      command will manually force that cache update to happen.
+
+
+## Architecture
+
+Dropsonde is a simple telemetry probe designed to run as a regular cron job. It
+will gather metrics defined by self-contained plugins that each defines its own
+partial schema and then gathers the data to meet that schema.
+
+Metrics plugins live in the `lib/dropsonde/metrics` directory. See the existing
+examples for the API. All data returned must match the schema the plugin defines
+and each column in the schema must be documented.
+
+Dropsonde maintains a cache listing all the public modules existing on the Forge.
+This is used for identifying public modules and ensuring that information about
+your internal modules is not leaked outside your network. Once a week, this cache
+of modules is updated.
+
+
+## Limitations
+
+This is super early in development and has not yet been battle tested.
+
+
+Contact
+-------
+
+community@puppet.com
+

data/bin/dropsonde

@@ -0,0 +1,84 @@
+#!/usr/bin/env ruby
+require 'gli'
+require 'dropsonde'
+require 'puppet'
+
+class Dropsonde
+  extend GLI::App
+
+  Puppet.initialize_settings
+
+  program_desc 'A simple telemetry tool for Puppet infrastructures'
+  config_file '/etc/puppetlabs/telemetry.yaml'
+
+  desc 'Verbose logging'
+  switch [:verbose, :v]
+
+  desc 'Auto update the Forge module name cache if expired'
+  switch [:update], :default_value => true
+
+  desc 'Path to cache directory'
+  flag [:cachepath], :default_value => "#{Puppet.settings[:vardir]}/dropsonde"
+
+  desc 'Forge module cache ttl in days'
+  flag [:ttl], :default_value => 7, :type => Integer
+
+  desc 'List of metrics to omit'
+  flag [:blacklist, :b], :type => Array
+
+  desc 'Any number or string used to generate the randomized site ID.'
+  flag [:seed]
+
+  pre do |global, command, options, args|
+    Dropsonde.settings = global
+    Dropsonde::Cache.init(global[:cachepath], global[:ttl], global[:update])
+  end
+
+  desc 'Manually update the Forge module name cache'
+  command :update do |c|
+    c.action do |global, options, args|
+      Dropsonde::Cache.update
+    end
+  end
+
+  desc 'Generate a complete schema set'
+  command :schema do |c|
+    c.action do |global, options, args|
+      Dropsonde.generate_schema
+    end
+  end
+
+  desc 'List all available metrics'
+  command :list do |c|
+    c.action do |global, options, args|
+      Dropsonde.list_metrics
+    end
+  end
+
+  desc 'Generate an example telemetry report'
+  command :preview do |c|
+    c.desc 'The output format to use'
+    c.flag [:format], :default_value => 'human'
+
+    c.action do |global, options, args|
+      Dropsonde::Cache.autoupdate
+      Dropsonde.generate_report(options[:format])
+    end
+  end
+
+  desc 'Submit a telemetry report'
+  command :submit do |c|
+    c.desc 'Telemetry endpoint'
+    c.flag [:endpoint], :default_value => 'https://dev.dujour.k8s.puppet.net'
+
+    c.desc 'Telemetry port'
+    c.flag [:port], :default_value => 443, :type => Integer
+
+    c.action do |global, options, args|
+      Dropsonde::Cache.autoupdate
+      Dropsonde.submit_report(options[:endpoint], options[:port])
+    end
+  end
+end
+
+exit Dropsonde.run(ARGV)

data/lib/dropsonde.rb

@@ -0,0 +1,84 @@
+require 'json'
+require 'httpclient'
+require 'puppetdb'
+require 'inifile'
+
+class Dropsonde
+  require 'dropsonde/cache'
+  require 'dropsonde/metrics'
+  require 'dropsonde/monkeypatches'
+
+  @@pdbclient = nil
+  @@settings  = {}
+  def self.settings=(arg)
+    raise "Requires a Hash to set all settings at once, not a #{arg.class}" unless arg.is_a? Hash
+    @@settings = arg
+  end
+
+  def self.settings
+    @@settings
+  end
+
+  def self.generate_schema
+    puts JSON.pretty_generate(Dropsonde::Metrics.new.schema)
+  end
+
+  def self.list_metrics
+    puts
+    puts Dropsonde::Metrics.new.list
+  end
+
+  def self.generate_report(format)
+    case format
+    when 'json'
+      puts JSON.pretty_generate(Dropsonde::Metrics.new.report)
+    when 'human'
+      puts
+      puts Dropsonde::Metrics.new.preview
+    else
+      raise "unknown format"
+    end
+  end
+
+  def self.submit_report(endpoint, port)
+    client = HTTPClient.new()
+    result = client.post("#{endpoint}:#{port}",
+                  :header => {'Content-Type' => 'application/json'},
+                  :body   => Dropsonde::Metrics.new.report.to_json
+                )
+
+    if result.status == 200
+      data = JSON.parse(result.body)
+      if data['newer']
+        puts 'A newer version of the telemetry client is available:'
+        puts "  -- #{data['link']}"
+      else
+        puts data['message']
+      end
+    else
+      puts 'Failed to submit report'
+      puts JSON.pretty_generate(result.body) if Dropsonde.settings[:verbose]
+      exit 1
+    end
+  end
+
+  def self.puppetDB
+    return @@pdbclient if @@pdbclient
+
+    config = File.join(Puppet.settings[:confdir], 'puppetdb.conf')
+
+    return unless File.file? config
+
+    server = IniFile.load(config)['main']['server_urls'].split(',').first
+
+    @@pdbclient = PuppetDB::Client.new({
+      :server => server,
+      :pem    => {
+          'key'     => Puppet.settings[:hostprivkey],
+          'cert'    => Puppet.settings[:hostcert],
+          'ca_file' => Puppet.settings[:localcacert],
+      }
+    })
+  end
+
+end

data/lib/dropsonde/cache.rb

@@ -0,0 +1,76 @@
+require 'date'
+require 'json'
+require 'fileutils'
+require 'puppet_forge'
+
+class Dropsonde::Cache
+  @@autoupdate = false
+
+  def self.init(path, ttl, autoupdate)
+    FileUtils.mkdir_p(path)
+    @@path = "#{File.expand_path(path)}/forge.json"
+    @@ttl  = ttl
+    @@autoupdate = autoupdate
+
+    if File.file? @@path
+      @@cache = JSON.parse(File.read(@@path))
+    else
+      @@cache = {
+                'timestamp' => '2000-1-1',  # long before any puppet modules were released!
+                'modules'   => [],
+              }
+    end
+
+    PuppetForge.user_agent = "Dropsonde Telemetry Client/0.0.1"
+  end
+
+  def self.modules
+    @@cache['modules']
+  end
+
+  def self.forgeModule?(mod)
+    case mod
+    when Puppet::Module
+      modname = mod.forge_slug
+    when Hash
+      modname = mod[:name] || mod['name']
+    when String
+      modname = mod
+    end
+    return unless modname
+
+    modules.include? modname.tr('/','-')
+  end
+
+  def self.update
+    iter   = PuppetForge::Module.all(:sort_by => 'latest_release')
+    newest = DateTime.parse(@@cache['timestamp'])
+
+    @@cache['timestamp'] = iter.first.created_at
+
+    until iter.next.nil?
+      # stop once we reach modules we've already cached
+      break if DateTime.parse(iter.first.created_at) <= newest
+
+      @@cache['modules'].concat iter.map {|mod| mod.slug }
+
+      iter = iter.next
+      print '.'
+    end
+    @@cache['modules'].sort!
+    @@cache['modules'].uniq!
+
+    File.write(@@path, JSON.pretty_generate(@@cache))
+  end
+
+  def self.autoupdate
+    return unless @@autoupdate
+
+    update unless File.file? @@path
+
+    if (Date.today - File.mtime(@@path).to_date).to_i > @@ttl
+      update
+    end
+  end
+
+end

data/lib/dropsonde/metrics.rb

@@ -0,0 +1,184 @@
+require 'little-plugger'
+
+class Dropsonde::Metrics
+  extend LittlePlugger( :path => 'dropsonde/metrics', :module => Dropsonde::Metrics)
+
+  def initialize
+    Dropsonde::Metrics.disregard_plugins(*Dropsonde.settings[:blacklist])
+    Dropsonde::Metrics.initialize_plugins
+  end
+
+  def siteid
+    return @siteid if @siteid
+
+    sha2 = Digest::SHA512.new
+    sha2.update Puppet.settings[:certname]
+    sha2.update Puppet.settings[:cacert]
+    sha2.update Dropsonde.settings[:seed] if Dropsonde.settings[:seed]
+    @siteid = sha2.hexdigest
+    @siteid
+  end
+
+  def list
+    str  = "                    Loaded telemetry plugins\n"
+    str << "                 ===============================\n\n"
+    Dropsonde::Metrics.plugins.each do |name, plugin|
+      str << name.to_s
+      str << "\n--------\n"
+      str << plugin.description.strip
+      str << "\n\n"
+    end
+    if Dropsonde.settings[:blacklist]
+      str << "Disabled plugins:\n"
+      str << "  #{Dropsonde.settings[:blacklist].join(', ')}"
+    end
+    str
+  end
+
+  def schema
+    schema = skeleton_schema
+    Dropsonde::Metrics.plugins.each do |name, plugin|
+      schema.concat(sanity_check_schema(plugin))
+    end
+    check_for_duplicates(schema)
+    schema
+  end
+
+  def preview
+    str  = "                      Puppet Telemetry Report Preview\n"
+    str << "                      ===============================\n\n"
+    Dropsonde::Metrics.plugins.each do |name, plugin|
+      schema = plugin.schema
+
+      plugin.setup
+      data = sanity_check_data(plugin)
+      plugin.cleanup
+
+      str << plugin.name+"\n"
+      str << "-------------------------------\n"
+      str << plugin.description
+      data.each do |row|
+        key    = row.keys.first
+        values = row.values.first
+
+        desc = schema.find {|item| item[:name].to_sym == key.to_sym}[:description]
+        str << "- #{key}: #{desc}\n"
+        values.each do |item|
+          str << "    #{item}\n"
+        end
+      end
+      str << "\n\n"
+    end
+    str << "Site ID:\n"
+    str << siteid
+    str
+  end
+
+  def report
+    snapshots = {}
+    Dropsonde::Metrics.plugins.each do |name, plugin|
+      plugin.setup
+      sanity_check_data(plugin).each do |row|
+        snapshots[row.keys.first] = {
+          'value'     => row.values.first,
+          'timestamp' => Time.now.iso8601,
+        }
+      end
+      plugin.cleanup
+    end
+
+    results = skeleton_report
+    results[:'self-service-analytics'][:snapshots] = snapshots
+    results
+  end
+
+  def sanity_check_data(plugin)
+    data = plugin.run
+    keys_data   = data.map {|item| item.keys }.flatten.map(&:to_s)
+    keys_schema = plugin.schema.map {|item| item[:name] }
+
+    disallowed = (keys_data - keys_schema)
+
+    raise "ERROR: The #{plugin.name} plugin exported the following keys not documented in the schema: #{disallowed}" unless disallowed.empty?
+
+    data
+  end
+
+  def sanity_check_schema(plugin)
+    schema = plugin.schema
+
+    if schema.class != Array or schema.find {|item| item.class != Hash}
+      raise "The #{plugin.name} plugin schema is not an array of hashes"
+    end
+
+    error = ''
+    [:name, :type, :description].each do |field|
+      count = schema.reject {|item| item[field] }.count
+      next if count == 0
+
+      error << "The #{plugin.name} plugin schema has #{count} missing #{field}s\n"
+    end
+    raise error unless error.empty?
+
+    schema
+  end
+
+  def check_for_duplicates(schema)
+    keys  = schema.map {|col| col[:name] }
+    dupes = keys.select{ |e| keys.count(e) > 1 }.uniq
+
+    raise "The schema defines duplicate keys: #{dupes}" unless dupes.empty?
+  end
+
+  def skeleton_schema
+    [
+      {
+        "description": "An ID that's unique for each checkin to Dujour.",
+        "mode": "NULLABLE",
+        "name": "message_id",
+        "type": "STRING"
+      },
+      {
+        "description": "A unique identifier for a site, derived as a hash of the CA certificate and optional seed.",
+        "mode": "NULLABLE",
+        "name": "site_id",
+        "type": "BYTES"
+      },
+      {
+        "description": "The name of the product.",
+        "mode": "NULLABLE",
+        "name": "product",
+        "type": "STRING"
+      },
+      {
+        "description": "Version of the project.",
+        "mode": "NULLABLE",
+        "name": "version",
+        "type": "STRING"
+      },
+      {
+        "description": "Time the checkin to Dujour occurred.",
+        "mode": "NULLABLE",
+        "name": "timestamp",
+        "type": "TIMESTAMP"
+      },
+      {
+        "description": "IP Address of node checking in to Dujour.",
+        "mode": "NULLABLE",
+        "name": "ip",
+        "type": "STRING"
+      }
+    ]
+  end
+
+  def skeleton_report
+    {
+      "product": "popularity-module",
+      "version": "1.0.0",
+      "site_id": siteid,
+      "self-service-analytics": {
+        "snapshots": { }
+      }
+    }
+  end
+end

data/lib/dropsonde/metrics/dependencies.rb

@@ -0,0 +1,70 @@
+class Dropsonde::Metrics::Dependencies
+  def self.initialize_dependencies
+    # require any libraries needed here -- no need to load puppet; it's already initialized
+    # All plugins are initialized before any metrics are generated.
+  end
+
+  def self.description
+    <<~EOF
+      This group of metrics discovers dependencies between modules in all
+      environments. It will omit dependencies on private modules.
+    EOF
+  end
+
+  def self.schema
+    # return an array of hashes of a partial schema to be merged into the complete schema
+    # See https://cloud.google.com/bigquery/docs/schemas#specifying_a_json_schema_file
+    [
+      {
+        "fields": [
+          {
+            "description": "The depended on module name",
+            "mode": "NULLABLE",
+            "name": "name",
+            "type": "STRING"
+          },
+          {
+            "description": "The depended on module version requirement",
+            "mode": "NULLABLE",
+            "name": "version_requirement",
+            "type": "STRING"
+          }
+        ],
+        "description": "List of modules that private modules in all environments depend on.",
+        "mode": "REPEATED",
+        "name": "dependencies",
+        "type": "RECORD"
+      }
+    ]
+  end
+
+  def self.setup
+    # run just before generating this metric
+  end
+
+  def self.run
+    # return an array of hashes representing the data to be merged into the combined checkin
+    environments = Puppet.lookup(:environments).list.map{|e|e.name}
+    modules = environments.map do |env|
+      Puppet.lookup(:environments).get(env).modules
+    end.flatten
+
+    # we want only PUBLIC modules that PRIVATE modules depend on
+    dependencies = modules.map do|mod|
+      next unless mod.dependencies
+      next if Dropsonde::Cache.forgeModule? mod  # skip unless this is a private module
+
+      # and return a list of all public modules it depends on
+      mod.dependencies.select {|mod| Dropsonde::Cache.forgeModule? mod }
+    end.flatten.compact
+
+    [
+      { :dependencies => dependencies },
+    ]
+
+  end
+
+  def self.cleanup
+    # run just after generating this metric
+  end
+end

data/lib/dropsonde/metrics/modules.rb

@@ -0,0 +1,114 @@
+class Dropsonde::Metrics::Modules
+  def self.initialize_modules
+    # require any libraries needed here -- no need to load puppet; it's already initialized
+    # All plugins are initialized before any metrics are generated.
+  end
+
+  def self.description
+    <<~EOF
+      This group of metrics exports name & version information about the public
+      modules installed in all environments, ignoring private modules.
+    EOF
+  end
+
+  def self.schema
+    # return an array of hashes of a partial schema to be merged into the complete schema
+    # See https://cloud.google.com/bigquery/docs/schemas#specifying_a_json_schema_file
+    [
+      {
+        "fields": [
+          {
+            "description": "The module name",
+            "mode": "NULLABLE",
+            "name": "name",
+            "type": "STRING"
+          },
+          {
+            "description": "The module slug (author-name)",
+            "mode": "NULLABLE",
+            "name": "slug",
+            "type": "STRING"
+          },
+          {
+            "description": "The module version",
+            "mode": "NULLABLE",
+            "name": "version",
+            "type": "STRING"
+          }
+        ],
+        "description": "List of modules in all environments.",
+        "mode": "REPEATED",
+        "name": "modules",
+        "type": "RECORD"
+      },
+      {
+        "fields": [
+          {
+            "description": "The class name",
+            "mode": "NULLABLE",
+            "name": "name",
+            "type": "STRING"
+          },
+          {
+            "description": "How many nodes it is declared on",
+            "mode": "NULLABLE",
+            "name": "count",
+            "type": "INTEGER"
+          }
+        ],
+        "description": "List of classes and counts in all environments.",
+        "mode": "REPEATED",
+        "name": "classes",
+        "type": "RECORD"
+      }
+    ]
+  end
+
+  def self.setup
+    # run just before generating this metric
+  end
+
+  def self.run
+    # return an array of hashes representing the data to be merged into the combined checkin
+    environments = Puppet.lookup(:environments).list.map{|e|e.name}
+    modules = environments.map do |env|
+      Puppet.lookup(:environments).get(env).modules.map do|mod|
+        next unless mod.forge_module?
+
+        {
+          :name    => mod.name,
+          :slug    => mod.forge_slug,
+          :version => mod.version,
+        }
+      end
+    end.flatten.compact.uniq
+
+    if Dropsonde.puppetDB
+      # classes and how many nodes they're enforced on
+      results = Dropsonde.puppetDB.request( '',
+        'resources[certname, type, title] { type = "Class" }'
+      ).data
+
+      # select only classes from public modules
+      classes = results.map do |klass|
+        next unless modules.find {|mod| mod[:name] == klass['title'].split('::').first.downcase }
+
+        {
+          :name  => klass['title'],
+          :count => results.count {|row| row['title'] == klass['title']},
+        }
+      end.compact.uniq
+    else
+      classes = []
+    end
+
+    [
+      { :modules => modules },
+      { :classes => classes },
+    ]
+  end
+
+  def self.cleanup
+    # run just after generating this metric
+  end
+end

data/lib/dropsonde/metrics/puppetfiles.rb

@@ -0,0 +1,76 @@
+class Dropsonde::Metrics::Puppetfiles
+  def self.initialize_puppetfiles
+    # require any libraries needed here -- no need to load puppet; it's already initialized
+    # All plugins are initialized before any metrics are generated.
+    require 'ripper'
+  end
+
+  def self.description
+    <<~EOF
+      This generates interesting stats about Puppetfiles used in your environments,
+      including whether your Puppetfiles have Ruby code in them.
+    EOF
+  end
+
+  def self.schema
+    # return an array of hashes of a partial schema to be merged into the complete schema
+    # See https://cloud.google.com/bigquery/docs/schemas#specifying_a_json_schema_file
+    [
+      {
+        "fields": [
+          {
+            "description": "The method name",
+            "mode": "NULLABLE",
+            "name": "name",
+            "type": "STRING"
+          },
+          {
+            "description": "How many times is it used",
+            "mode": "NULLABLE",
+            "name": "count",
+            "type": "INTEGER"
+          }
+        ],
+        "description": "Ruby methods used in Puppetfiles.",
+        "mode": "REPEATED",
+        "name": "puppetfile_ruby_methods",
+        "type": "RECORD"
+      }
+    ]
+  end
+
+  def self.setup
+    # run just before generating this metric
+  end
+
+  def self.run
+    methods = Dir.entries(Puppet.settings[:environmentpath]).map do |entry|
+      puppetfile = File.join(Puppet.settings[:environmentpath], entry, 'Puppetfile')
+
+      next if entry.start_with? '.'
+      next unless File.file? puppetfile
+
+      tokens  = Ripper.sexp(File.read(puppetfile)).flatten
+      indices = tokens.map.with_index {|a, i| a == :command ? i : nil}.compact
+
+      indices.map {|i| tokens[i+2] }
+    end.flatten.compact
+
+    methods.reject! {|name| ['mod', 'forge', 'moduledir'].include? name }
+
+    methods = methods.uniq.map do |name|
+      {
+        :name  => name,
+        :count => methods.count(name),
+      }
+    end
+
+    [
+      { :puppetfile_ruby_methods => methods },
+    ]
+  end
+
+  def self.cleanup
+    # run just after generating this metric
+  end
+end

data/lib/dropsonde/monkeypatches.rb

@@ -0,0 +1,17 @@
+module Puppet
+  class Module
+
+    unless Module.method_defined? :"forge_module?"
+      def forge_module?
+        Dropsonde::Cache.forgeModule? self
+      end
+    end
+
+    unless Module.method_defined? :forge_slug
+      def forge_slug
+        self.forge_name.tr('/','-') rescue nil
+      end
+    end
+
+  end
+end

metadata

@@ -0,0 +1,182 @@
+--- !ruby/object:Gem::Specification
+name: dropsonde
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Ben Ford
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-03-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: gli
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: httpclient
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: little-plugger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: puppet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: puppet_forge
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: semantic_puppet
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: inifile
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: puppetdb-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: |
+  Dropsonde is a simple telemetry probe designed to run as a regular cron job. It
+  will gather metrics defined by self-contained plugins that each defines its own
+  partial schema and then gathers the data to meet that schema.
+email: ben.ford@puppet.com
+executables:
+- dropsonde
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE
+- README.md
+- bin/dropsonde
+- lib/dropsonde.rb
+- lib/dropsonde/cache.rb
+- lib/dropsonde/metrics.rb
+- lib/dropsonde/metrics/dependencies.rb
+- lib/dropsonde/metrics/modules.rb
+- lib/dropsonde/metrics/puppetfiles.rb
+- lib/dropsonde/monkeypatches.rb
+homepage: 
+licenses:
+- Apache 2
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.0.3
+signing_key: 
+specification_version: 4
+summary: A simple telemetry probe for gathering usage information about Puppet infrastructures.
+test_files: []