puppet-community-mvp 0.0.3 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: f77ade2721786fdca7fd96827ca510e0404a02a8
-  data.tar.gz: ad32f5a43392ed7f23d49d60db3fbe308ae73e2d
+SHA256:
+  metadata.gz: dd83202b003a900b8744b0fc8da5bb14b6024ca37f1419c841d68afaa4b487dd
+  data.tar.gz: c69cfa9c035b30136593d10dbad96d588fd0fe8dd870ba0763068f0c756af5cc
 SHA512:
-  metadata.gz: be5282a77000b433c3aedd58ddfd29ea594df089007e38e74cb8b18c5c6b576ff048b218ec30cb9f60b8291faa1b62525f8d2c79694444bb3f95df2ede98efc0
-  data.tar.gz: 94894492ffba9d187f9178a17d51e412d80dc7b3434681bf43da79f422c6bbb0ddddba7f347b49ed048e0134a72eefe5ff3d83a615d5c4402d809f3e8010c1ac
+  metadata.gz: 5370badaaa4208281fa6e864a398482cd6403aeacaa8ef1ec7ac661d39287cff944a483c3f16ed352feb27b3c230573d627ebf75c43d5be4d496313553706f11
+  data.tar.gz: 1418192f0b6adc010b7982b34c2b8f1654b6a3fa6fbd88533a2bf766aea2a4ad3ffb63ba6d3d3481c67c1cf74fa0ec9796dc520474558750bda62fd32d05d24a

data/README.md

@@ -0,0 +1,78 @@
+# Puppet Community MVP tool
+
+This is a simple tool to generate stats about the Puppet community. It was
+originally intended to show the "most valuable players" but has since morphed to
+show a lot of other things too. We primarily use it on a weekly cron job to
+gather information using the Forge APIs and normalizing them so that they can be
+easily combined with simple SQL queries to generate usage information.
+
+## Interactive usage
+
+If you're not working on our community stats pipeline, then there are only three
+subcommands you'll be interested in.
+
+### `stats`
+
+This subcommand will use cached data to generate a report of Forge community
+statistics. For example, it will generate distributions of module quality
+scores, or releases per module, or modules per author, etc. And it will generate
+sparklines showing the contributions over time of the most prolific Forge
+authors and it will show authors who aren't as active as they used to be.
+
+Unfortunately, this report is not customizable or templatable at this point.
+
+You will need cached data before you can generate this report. See the `get` subcommand.
+
+
+### `get`
+
+This subcommand will download and cache a local mirror of the data stored in our
+BigQuery database. This data is used for the `stats` command.
+
+
+### `analyze`
+
+This subcommand is maybe the most interesting. Many interesting bits of
+information can be gathered by inspecting the source code of modules, not by
+running SQL queries about their statistics. For example, `find manifests/ -name
+'*.pp' | wc -l` will tell you how many manifests any given module includes, and
+`grep -rn '--no-external-facts' facts.d/` will tell you how many external facts
+are invoking `facter` to gather and use _other_ facts while running.
+
+This command lets you write that little bit of analysis code as a script, and
+then systematically run that script against the current release of every single
+module on the Forge and collate the generated output.
+
+A script can be written in any language and will be executed from the root of
+the unpacked module. It will be invoked with an environment containing the following
+variables:
+
+* `mvp_owner` -- the Forge namespace of the module, aka the author's username
+* `mvp_name` -- the name of the module itself
+* `mvp_version` -- the current version of the module
+* `mvp_downloads` -- the number of downloads this module has. A *rough* estimation of popularity
+
+The script should print an array of arrays in JSON format to STDOUT. These will be
+combined to make a CSV file, the columns of which are defined by the data you
+return. In other words, the items in the inner array(s) are totally up to you.
+They will become the columns of the generated CSV file.
+
+The parameters relevant to this subcommand are:
+
+```
+    -o, --output_file OUTPUT_FILE    The path to save a csv report.
+        --script SCRIPT              The script file to analyze a module. See docs for interface.
+        --count N                    For debugging. Select a random list of this many modules to analyze.
+    -d, --debug                      Display extra debugging information.
+```
+
+See files in the `scripts/` directory for examples of analysis scripts. To use,
+just path of a script, like
+
+```
+$ mvp analyze --script scripts/manifest_count.rb --count 5
+[✔] stdlib (OK)
+$ cat analyzed.csv
+...
+```
+

data/bin/mvp

@@ -13,19 +13,21 @@ optparse = OptionParser.new { |opts|
   opts.banner = "Usage : #{NAME} [command] [target] [options]
 
 This tool will scrape the Puppet Forge API for interesting module & author stats.
-The following CLI commands are available.
+It can also mirror public BigQuery tables or views into our dataset for efficiency,
+or download and itemize each Forge module.
 
-  * get | retrieve | download [target]
-      * Downloads and caches all Forge metadata.
-      * Optional targets: all, authors, modules, releases
-  * upload | insert [target]
-      * Uploads data to BigQuery
-      * Optional targets: all, authors, modules, releases, mirrors
   * mirror [target]
       * Runs the download & then upload tasks.
+      * Optional targets: all, authors, modules, releases, validations, itemizations, puppetfiles, tables
+  * get | retrieve | download [target]
+      * Downloads and caches data locally so you can run the stats task.
       * Optional targets: all, authors, modules, releases
   * stats
       * Print out a summary of interesting stats.
+  * analyze <script file>
+      * Run a specified script to analyze each module to generate arbitrary stats
+      * Writes output to a csv file, analyzed.csv by default
+
 "
 
   opts.on("-f FORGEAPI", "--forgeapi FORGEAPI", "Forge API server. Rarely needed.") do |arg|
@@ -60,10 +62,22 @@ The following CLI commands are available.
     options[:output_file] = arg
   end
 
+  opts.on("--script SCRIPT", "The script file to analyze a module. See docs for interface.") do |arg|
+    options[:script] = arg
+  end
+
+  opts.on("--count N", "For debugging. Select a random list of this many modules to analyze.") do |arg|
+    options[:count] = arg.to_i
+  end
+
   opts.on("-d", "--debug", "Display extra debugging information.") do
     options[:debug] = true
   end
 
+  opts.on("-n", "--noop", "Don't actually upload data.") do
+    options[:noop] = true
+  end
+
   opts.separator('')
 
   opts.on("-h", "--help", "Displays this help") do
@@ -83,31 +97,29 @@ options[:gcloud][:dataset] ||= 'community'
 options[:gcloud][:project] ||= 'puppet'
 options[:gcloud][:keyfile] ||= '~/.mvp/credentials.json'
 
+options[:script]             = File.expand_path(options[:script]) if options[:script]
 options[:cachedir]           = File.expand_path(options[:cachedir])
 options[:github_data]        = File.expand_path(options[:github_data])
 options[:gcloud][:keyfile]   = File.expand_path(options[:gcloud][:keyfile])
 FileUtils.mkdir_p(options[:cachedir])
 
+command, target = ARGV
+case command
+when 'analyze'
+  options[:output_file] ||= 'analyzed.csv'
+end
+
 $logger           = Logger::new(STDOUT)
 $logger.level     = options[:debug] ? Logger::DEBUG : Logger::INFO
 $logger.formatter = proc { |severity,datetime,progname,msg| "#{severity}: #{msg}\n" }
 
 runner = Mvp::Runner.new(options)
 
-command, target = ARGV
 case command
 when 'get', 'retrieve', 'download'
   target ||= :all
   runner.retrieve(target.to_sym)
 
-when 'transform'
-  target ||= :all
-  runner.retrieve(target.to_sym, false)
-
-when 'insert', 'upload'
-  target ||= :all
-  runner.upload(target.to_sym)
-
 when 'mirror'
   target ||= :all
   runner.mirror(target.to_sym)
@@ -116,6 +128,9 @@ when 'stats'
   target ||= :all
   runner.stats(target.to_sym)
 
+when 'analyze'
+  runner.analyze
+
 when 'test'
   runner.test
 

data/bin/pftest.rb

@@ -0,0 +1,22 @@
+#! /usr/bin/env ruby
+
+require 'mvp/puppetfile_parser'
+require 'open-uri'
+require 'json'
+require 'logger'
+
+$logger           = Logger::new(STDOUT)
+$logger.level     = Logger::INFO
+$logger.formatter = proc { |severity,datetime,progname,msg| "#{severity}: #{msg}\n" }
+
+pf = open(ARGV.first)
+parser = Mvp::PuppetfileParser.new()
+
+
+repo = {
+  :repo_name => 'testing',
+  :md5       => 'wakka wakka',
+  :content   => pf.read,
+}
+
+puts JSON.pretty_generate(parser.parse(repo))

data/lib/mvp.rb

@@ -1,4 +1,2 @@
 require 'mvp/runner'
-require 'mvp/downloader'
-require 'mvp/uploader'
-require 'mvp/stats'
+require 'mvp/stats'

data/lib/mvp/{uploader.rb → bigquery.rb}

@@ -3,10 +3,10 @@ require 'tty-spinner'
 require "google/cloud/bigquery"
 
 class Mvp
-  class Uploader
+  class Bigquery
     def initialize(options = {})
+      @options  = options
       @cachedir = options[:cachedir]
-      @mirrors  = options[:gcloud][:mirrors]
       @bigquery = Google::Cloud::Bigquery.new(
         :project_id  => options[:gcloud][:project],
         :credentials => Google::Cloud::Bigquery::Credentials.new(options[:gcloud][:keyfile]),
@@ -16,7 +16,7 @@ class Mvp
       raise "\nThere is a problem with the gCloud configuration: \n #{JSON.pretty_generate(options)}" if @dataset.nil?
 
       @itemized = @dataset.table('forge_itemized') || @dataset.create_table('forge_itemized') do |table|
-                                                        table.name        = 'Itemied dependencies between modules'
+                                                        table.name        = 'Itemized dependencies between modules'
                                                         table.description = 'A list of all types/classes/functions used by each module and where they come from'
                                                         table.schema do |s|
                                                           s.string  "module",  mode: :required
@@ -27,9 +27,24 @@ class Mvp
                                                           s.integer "count",   mode: :required
                                                         end
                                                       end
+
+      @puppetfile_usage = @dataset.table('github_puppetfile_usage') || @dataset.create_table('github_puppetfile_usage') do |table|
+                                                                          table.name        = 'Puppetfile Module Usage'
+                                                                          table.description = 'A list of all modules referenced in public Puppetfiles'
+                                                                          table.schema do |s|
+                                                                            s.string    "repo_name", mode: :required
+                                                                            s.string    "module",    mode: :required
+                                                                            s.string    "type",      mode: :required
+                                                                            s.string    "source"
+                                                                            s.string    "version"
+                                                                            s.string    "md5",       mode: :required
+                                                                          end
+                                                                        end
     end
 
     def truncate(entity)
+      return if @options[:noop]
+
       begin
         case entity
         when :authors
@@ -65,6 +80,7 @@ class Mvp
               s.timestamp "created_at",       mode: :required
               s.timestamp "updated_at",       mode: :required
               s.string    "tasks",            mode: :repeated
+              s.string    "plans",            mode: :repeated
               s.string    "homepage_url"
               s.string    "project_page"
               s.string    "issues_url"
@@ -125,6 +141,7 @@ class Mvp
               s.timestamp "deleted_at"
               s.string    "deleted_for"
               s.string    "tasks",            mode: :repeated
+              s.string    "plans",            mode: :repeated
               s.string    "project_page"
               s.string    "issues_url"
               s.string    "source"
@@ -144,11 +161,9 @@ class Mvp
               s.boolean   "puppet_99x"
               s.string    "dependencies",     mode: :repeated
               s.string    "file_uri",         mode: :required
-              s.string    "file_md5",         mode: :required
+              s.string    "file_md5"
+              s.string    "file_sha256"
               s.integer   "file_size",        mode: :required
-              s.string    "changelog"
-              s.string    "reference"
-              s.string    "readme"
               s.string    "license"
               s.string    "metadata",         mode: :required
             end
@@ -163,99 +178,90 @@ class Mvp
       end
     end
 
-    def authors()
-      upload('authors')
-    end
-
-    def modules()
-      upload('modules')
+    def retrieve(entity)
+      get(entity, ['*'])
     end
 
-    def releases()
-      upload('releases')
-    end
+    def mirror_table(entity)
+      return if @options[:noop]
 
-    def validations()
-      upload('validations')
-    end
-
-    def github_mirrors()
-      @mirrors.each do |entity|
-        begin
-          spinner = TTY::Spinner.new("[:spinner] :title")
-          spinner.update(title: "Mirroring #{entity[:type]} #{entity[:name]} to BigQuery...")
-          spinner.auto_spin
-
-          case entity[:type]
-          when :view
-            @dataset.table(entity[:name]).delete rescue nil # delete if exists
-            @dataset.create_view(entity[:name], entity[:query],
-                                  :legacy_sql => true)
-
-          when :table
-            job = @dataset.query_job(entity[:query],
-                                  :legacy_sql => true,
-                                  :write      => 'truncate',
-                                  :table      => @dataset.table(entity[:name], :skip_lookup => true))
-            job.wait_until_done!
+      begin
+        case entity[:type]
+        when :view
+          @dataset.table(entity[:name]).delete rescue nil # delete if exists
+          @dataset.create_view(entity[:name], entity[:query])
 
-          else
-            $logger.error "Unknown mirror type: #{entity[:type]}"
-          end
+        when :table
+          job = @dataset.query_job(entity[:query],
+                                :write      => 'truncate',
+                                :table      => @dataset.table(entity[:name], :skip_lookup => true))
+          job.wait_until_done!
 
-          spinner.success('(OK)')
-        rescue => e
-          spinner.error("(Google Cloud error: #{e.message})")
-          $logger.error e.backtrace.join("\n")
+        else
+          $logger.error "Unknown mirror type: #{entity[:type]}"
         end
+      rescue => e
+        $logger.error("(Google Cloud error: #{e.message})")
+        $logger.debug e.backtrace.join("\n")
       end
     end
 
-    def insert(entity, data)
-      table    = @dataset.table("forge_#{entity}")
+    def insert(entity, data, suite = 'forge')
+      return if @options[:noop]
+      return if data.empty?
+
+      table    = @dataset.table("#{suite}_#{entity}")
       response = table.insert(data)
 
       unless response.success?
-        errors = {}
+        $logger.error '========================================================================='
         response.insert_errors.each do |err|
-          errors[err.row['slug']] = err.errors
+          $logger.debug JSON.pretty_generate(err.row.reject {|k,v| ['metadata'].include? k})
+          $logger.error JSON.pretty_generate(err.errors)
         end
-        $logger.error JSON.pretty_generate(errors)
       end
     end
 
-    def upload(entity)
-      begin
-        spinner = TTY::Spinner.new("[:spinner] :title")
-        spinner.update(title: "Uploading #{entity} to BigQuery ...")
-        spinner.auto_spin
+    def delete(entity, field, match, suite = 'forge')
+      @dataset.query("DELETE FROM #{suite}_#{entity} WHERE #{field} = '#{match}'")
+    end
 
-        @dataset.load("forge_#{entity}", "#{@cachedir}/nld_#{entity}.json",
-                        :write      => 'truncate',
-                        :autodetect => true)
+    def get(entity, fields, suite = 'forge')
+      raise 'pass fields as an array' unless fields.is_a? Array
+      @dataset.query("SELECT #{fields.join(', ')} FROM #{suite}_#{entity}")
+    end
 
-#         table = @dataset.table("forge_#{entity}")
-#         File.readlines("#{@cachedir}/nld_#{entity}.json").each do |line|
-#           data = JSON.parse(line)
-#
-#           begin
-#             table.insert data
-#           rescue
-#             require 'pry'
-#             binding.pry
-#           end
-#         end
+    def module_sources()
+      get('modules', ['slug', 'source'])
+    end
 
+    def puppetfiles()
+      sql = 'SELECT f.repo_name, f.path, c.content, c.md5
+                FROM github_puppetfile_files AS f
+                JOIN github_puppetfile_contents AS c
+                  ON c.id = f.id
 
-        spinner.success('(OK)')
-      rescue => e
-        spinner.error("(Google Cloud error: #{e.message})")
-        $logger.error e.backtrace.join("\n")
-      end
+              WHERE c.md5 NOT IN (
+                SELECT u.md5
+                FROM github_puppetfile_usage AS u
+                WHERE u.repo_name = f.repo_name
+              ) AND LOWER(repo_name) NOT LIKE "%boxen%"'
+      @dataset.query(sql)
+    end
+
+    def unitemized()
+      sql = 'SELECT m.name, m.slug, m.version, m.dependencies
+              FROM forge_modules AS m
+              WHERE m.version NOT IN (
+                SELECT i.version
+                FROM forge_itemized AS i
+                WHERE module = m.slug
+              )'
+      @dataset.query(sql)
     end
 
     def version_itemized?(mod, version)
-      str = "SELECT version FROM forge_itemized WHERE name = '#{mod}' UNIQUE"
+      str = "SELECT DISTINCT version FROM forge_itemized WHERE module = '#{mod}'"
       versions = @dataset.query(str).map {|row| row[:version] } rescue []
 
       versions.include? version

data/lib/mvp/{downloader.rb → forge.rb}

@@ -2,151 +2,82 @@ require 'json'
 require 'httparty'
 require 'tty-spinner'
 require 'semantic_puppet'
-require 'mvp/monkeypatches'
-require 'mvp/itemizer'
 
 class Mvp
-  class Downloader
+  class Forge
     def initialize(options = {})
       @useragent = 'Puppet Community Stats Monitor'
-      @cachedir  = options[:cachedir]
       @forgeapi  = options[:forgeapi] ||'https://forgeapi.puppet.com'
-      @itemizer  = Mvp::Itemizer.new(options)
     end
 
-    def mirror(entity, uploader)
-      # using authors for git repo terminology consistency
-      item = (entity == :authors) ? 'users' : entity.to_s
-      download(item) do |data|
-        case entity
-        when :modules
-          uploader.insert(:validations, flatten_validations(retrieve_validations(data)))
-          data = flatten_modules(data)
-
-          @itemizer.run!(data, uploader)
-        when :releases
-          data = flatten_releases(data)
-        end
-
-        uploader.insert(entity, data)
-      end
-    end
-
-    def retrieve(entity, download = true)
-      if download
-        # I am focusing on authorship rather than just users, so for now I'm using the word authors
-        item = (entity == :authors) ? 'users' : entity.to_s
-        data = []
-        download(item) do |resp|
-          data.concat resp
-        end
-        save_json(entity, data)
-      else
-        data = File.read("#{@cachedir}/#{entity}.json")
-      end
-
-      case entity
-      when :modules
-        data = flatten_modules(data)
-      when :releases
-        data = flatten_releases(data)
-      end
-      save_nld_json(entity.to_s, data)
-    end
-
-    def retrieve_validations(modules, period = 25)
-      results = {}
+    def retrieve(entity)
+      raise 'Please process downloaded data by passing a block' unless block_given?
 
+      # using authors for git repo terminology consistency
+      entity = :users if entity == :authors
       begin
         offset   = 0
-        endpoint = "/private/validations/"
-        modules.each do |mod|
-          name = "#{mod['owner']['username']}-#{mod['name']}"
-          response = HTTParty.get("#{@forgeapi}#{endpoint}#{name}", headers: {'User-Agent' => @useragent})
+        endpoint = "/v3/#{entity}?sort_by=downloads&limit=50"
+
+        while endpoint do
+          response = HTTParty.get("#{@forgeapi}#{endpoint}", headers: {"User-Agent" => @useragent})
           raise "Forge Error: #{@response.body}" unless response.code == 200
+          data    = JSON.parse(response.body)
+          results = munge_dates(data['results'])
+
+          case entity
+          when :modules
+            results = flatten_modules(results)
+          when :releases
+            results = flatten_releases(results)
+          end
 
-          results[name] = JSON.parse(response.body)
-          offset       += 1
+          yield results, offset
 
-          if block_given? and (offset % period == 0)
-            yield offset
+          offset  += 50
+          endpoint = data['pagination']['next']
+          if (endpoint and (offset % 250 == 0))
             GC.start
           end
         end
+
       rescue => e
         $logger.error e.message
         $logger.debug e.backtrace.join("\n")
       end
 
-      results
+      nil
     end
 
-    def validations()
-      cache = "#{@cachedir}/modules.json"
-
-      if File.exist? cache
-        module_data = JSON.parse(File.read(cache))
-      else
-        module_data = retrieve(:modules)
-      end
+    def retrieve_validations(modules, period = 25)
+      raise 'Please process validations by passing a block' unless block_given?
 
+      offset = 0
       begin
-        spinner = TTY::Spinner.new("[:spinner] :title")
-        spinner.update(title: "Downloading module validations ...")
-        spinner.auto_spin
+        modules.each_slice(period) do |group|
+          offset += period
+          results = group.map { |mod| validations(mod[:slug]) }
 
-        results = retrieve_validations(module_data) do |offset|
-          spinner.update(title: "Downloading module validations [#{offset}]...")
+          yield results, offset
+          GC.start
         end
-
-        spinner.success('(OK)')
       rescue => e
-        spinner.error('API error')
         $logger.error e.message
         $logger.debug e.backtrace.join("\n")
       end
 
-      save_json('validations', results)
-      save_nld_json('validations', flatten_validations(results))
-      results
+      nil
     end
 
-    def download(entity)
-       raise 'Please process downloaded data by passing a block' unless block_given?
-
-      begin
-        offset   = 0
-        endpoint = "/v3/#{entity}?sort_by=downloads&limit=50"
-        spinner  = TTY::Spinner.new("[:spinner] :title")
-        spinner.update(title: "Downloading #{entity} ...")
-        spinner.auto_spin
-
-        while endpoint do
-          response = HTTParty.get("#{@forgeapi}#{endpoint}", headers: {"User-Agent" => @useragent})
-          raise "Forge Error: #{@response.body}" unless response.code == 200
-          data = JSON.parse(response.body)
-
-          offset  += 50
-          endpoint = data['pagination']['next']
-
-          yield munge_dates(data['results'])
-
-          if (endpoint and (offset % 250 == 0))
-            spinner.update(title: "Downloading #{entity} [#{offset}]...")
-            GC.start
-          end
-        end
-
-        spinner.success('(OK)')
-      rescue => e
-        spinner.error('API error')
-        $logger.error e.message
-        $logger.debug e.backtrace.join("\n")
-      end
+    def validations(name)
+      endpoint = "/private/validations/"
+      response = HTTParty.get("#{@forgeapi}#{endpoint}#{name}", headers: {'User-Agent' => @useragent})
+      raise "Forge Error: #{@response.body}" unless response.code == 200
 
-      nil
+      flatten_validations(name, JSON.parse(response.body))
     end
 
+
     # transform dates into a format that bigquery knows
     def munge_dates(object)
       ["created_at", "updated_at", "deprecated_at", "deleted_at"].each do |field|
@@ -160,16 +91,6 @@ class Mvp
       object
     end
 
-    def save_json(thing, data)
-      File.write("#{@cachedir}/#{thing}.json", data.to_json)
-    end
-
-    # store data in a way that bigquery can grok
-    # uploading files is far easier than streaming data, when replacing a dataset
-    def save_nld_json(thing, data)
-      File.write("#{@cachedir}/nld_#{thing}.json", data.to_newline_delimited_json)
-    end
-
     def flatten_modules(data)
       data.each do |row|
         row['owner']             = row['owner']['username']
@@ -183,6 +104,7 @@ class Mvp
         row['project_page']      = row['current_release']['metadata']['project_page']
         row['issues_url']        = row['current_release']['metadata']['issues_url']
         row['tasks']             = row['current_release']['tasks'].map{|task| task['name']} rescue []
+        row['plans']             = row['current_release']['plans'].map{|task| task['name']} rescue []
 
         row['release_count']     = row['releases'].count rescue 0
         row['releases']          = row['releases'].map{|r| r['version']} rescue []
@@ -202,21 +124,24 @@ class Mvp
         row['project_page']      = row['metadata']['project_page']
         row['issues_url']        = row['metadata']['issues_url']
         row['tasks']             = row['tasks'].map{|task| task['name']} rescue []
+        row['plans']             = row['plans'].map{|task| task['name']} rescue []
 
         simplify_metadata(row, row['metadata'])
-        row.delete('module')
+
+        # These items are just too big to store in the table, and the malware scan isn't done yet
+        ['module', 'changelog', 'readme', 'reference', 'malware_scan'].each do |column|
+          row.delete(column)
+        end
       end
       data
     end
 
-    def flatten_validations(data)
-      data.map do |name, scores|
-        row = { 'name' => name }
-        scores.each do |entry|
-          row[entry['name']] = entry['score']
-        end
-        row
+    def flatten_validations(name, scores)
+      row = { 'name' => name }
+      scores.each do |entry|
+        row[entry['name']] = entry['score']
       end
+      row
     end
 
     def simplify_metadata(data, metadata)

data/lib/mvp/itemizer.rb

@@ -12,7 +12,7 @@ class Mvp
 
     def run!(data, uploader)
       data.each do |mod|
-        modname = mod['slug']
+        modname = mod['name']
         version = mod['version']
         return if uploader.version_itemized?(modname, version)
 
@@ -27,13 +27,23 @@ class Mvp
       end
     end
 
+    def itemized(mod)
+      modname = mod[:slug]
+      version = mod[:version]
+      baserow = { :module => modname, :version => version, :kind => 'admin', :element => 'version', :count => 0}
+
+      table(itemize(modname, version), mod) << baserow
+    end
+
     def download(path, modname, version)
       filename = "#{modname}-#{version}.tar.gz"
       Dir.chdir(path) do
         File.open(filename, "w") do |file|
           file << HTTParty.get( "#{@forge}/v3/files/#{filename}" )
         end
-        system("tar -xf #{filename}")
+        # Why is tar terrible?
+        FileUtils.mkdir("#{modname}-#{version}")
+        system("tar -xf #{filename} -C #{modname}-#{version} --strip-components=1")
         FileUtils.rm(filename)
       end
     end
@@ -55,23 +65,67 @@ class Mvp
       end
     end
 
+    def analyze(mod, script, debug)
+      require 'open3'
+      require 'json'
+
+      # sanitize an environment
+      env = {'mvp_script' => script}
+      mod.each do |key, value|
+        env["mvp_#{key}"] = value.to_s
+      end
+
+      downloads = mod[:downloads]
+      Dir.mktmpdir('mvp') do |path|
+        download(path, "#{mod[:owner]}-#{mod[:name]}", mod[:version])
+
+        rows = []
+        Dir.chdir("#{path}/#{mod[:owner]}-#{mod[:name]}-#{mod[:version]}") do
+          if debug
+            exit(1) unless system(env, ENV['SHELL'])
+          end
+
+          stdout, stderr, status = Open3.capture3(env, script)
+
+          if status.success?
+            rows = JSON.parse(stdout)
+          else
+            $logger.error stderr
+          end
+        end
+
+        return rows unless rows.empty?
+      end
+    end
+
     # Build a table with this schema
     # module | version | source | kind | element | count
     def table(itemized, data)
-      modname      = data['slug']
-      version      = data['version']
-      dependencies = data['dependencies']
+      modname      = data[:name]
+      slug         = data[:slug]
+      version      = data[:version]
+      dependencies = data[:dependencies]
 
       itemized.map do |kind, elements|
         # the kind of element comes pluralized from puppet-itemize
         kind = kind.to_s
         kind = kind.end_with?('ses') ? kind.chomp('es') : kind.chomp('s')
         elements.map do |name, count|
-          # TODO: this may suffer from collisions, (module foo, function foo, for example)
-          depname = name.split('::').first
+          if name == modname
+            depname = name
+          else
+            # This relies on a little guesswork.
+            segments = name.split('::')                       # First see if its already namespaced and we can just use it
+            segments = name.split('_') if segments.size == 1  # If not, then maybe it follows the pattern like 'mysql_password'
+            depname  = segments.first
+          end
+
+          # There's a chance of collisions here. For example, if you depended on a module
+          # named 'foobar-notify' and you used a 'notify' resource, then the resource would
+          # be improperly linked to that module. That's a pretty small edge case though.
           source  = dependencies.find {|row| row.split('-').last == depname} rescue nil
 
-          { :module => modname, :version => version, :source => source, :kind => kind, :element => name, :count => count }
+          { :module => slug, :version => version, :source => source, :kind => kind, :element => name, :count => count }
         end
       end.flatten(1)
     end

data/lib/mvp/puppetfile_parser.rb

@@ -0,0 +1,171 @@
+class Mvp
+  class PuppetfileParser
+    def initialize(options = {})
+      @sources = {}
+      @modules = []
+      @repo    = nil
+    end
+
+    def suitable?
+      defined?(RubyVM::AbstractSyntaxTree)
+    end
+
+    def sources=(modules)
+      modules.each do |row|
+        next unless row[:source]
+        next if row[:source] == 'UNKNOWN'
+
+        @sources[canonical_git_repo(row[:source])] = row[:slug]
+      end
+    end
+
+    def parse(repo)
+      # This only works on Ruby 2.6+
+      return unless suitable?
+
+      begin
+        root = RubyVM::AbstractSyntaxTree.parse(repo[:content])
+      rescue SyntaxError => e
+        $logger.warn "Syntax error in #{repo[:repo_name]}/Puppetfile"
+        $logger.warn e.message
+      end
+
+      @repo    = repo
+      @modules = []
+      traverse(root)
+      @modules.compact.map do |row|
+        row[:repo_name] = repo[:repo_name]
+        row[:md5]       = repo[:md5]
+        row[:module]    = canonical_name(row[:module], row[:source])
+        stringify(row)
+      end
+    end
+
+    def stringify(row)
+      row.each do |key, value|
+        if value.is_a? RubyVM::AbstractSyntaxTree::Node
+          row[key] = :'#<programmatically generated via ruby code>'
+        end
+      end
+    end
+
+    def canonical_name(name, repo)
+      return name if name.include?('-')
+      repo = canonical_git_repo(repo)
+
+      return @sources[repo] if @sources.include?(repo)
+      name
+    end
+
+    def canonical_git_repo(repo)
+      return unless repo
+      return unless repo.is_a? String
+      repo.sub(/^git@github.com\:/, 'github.com/')
+          .sub(/^(git|https?)\:\/\//, '')
+          .sub(/\.git$/, '')
+    end
+
+    def add_module(name, args)
+      unless name.is_a? String
+        $logger.warn "Non string module name in #{@repo[:repo_name]}/Puppetfile"
+        return nil
+      end
+      name.gsub!('/', '-')
+      case args
+      when String, Symbol, NilClass
+        @modules << {
+          :module  => name,
+          :type    => :forge,
+          :source  => :forge,
+          :version => args,
+        }
+      when Hash
+        @modules << parse_args(name, args)
+      else
+        $logger.warn "#{@repo[:repo_name]}/Puppetfile: Unknown format: mod('#{name}', #{args.inspect})"
+      end
+    end
+
+    def parse_args(name, args)
+      data = {:module => name}
+
+      if args.include? :git
+        data[:type]    = :git
+        data[:source]  = args[:git]
+        data[:version] = args[:ref] || args[:tag] || args[:commit] || args[:branch] || :latest
+      elsif args.include? :svn
+        data[:type]    = :svn
+        data[:source]  = args[:svn]
+        data[:version] = args[:rev] || args[:revision] || :latest
+      elsif args.include? :boxen
+        data[:type]    = :boxen
+        data[:source]  = args[:repo]
+        data[:version] = args[:version] || :latest
+      else
+        $logger.warn "#{@repo[:repo_name]}/Puppetfile: Unknown args format: mod('#{name}', #{args.inspect})"
+        return nil
+      end
+
+      data
+    end
+
+    def traverse(node)
+      begin
+        if node.type == :FCALL
+          name = node.children.first
+          args = node.children.last.children.map do |item|
+            next if item.nil?
+
+            case item.type
+            when :HASH
+              Hash[*item.children.first.children.compact.map {|n| n.children.first }]
+            else
+              item.children.first
+            end
+          end.compact
+
+          case name
+          when :mod
+            add_module(args.shift, args.shift)
+          when :forge
+            # noop
+          when :moduledir
+            # noop
+          when :github
+            # oh boxen, you so silly.
+            # The order of the unpacking below *is* important.
+            modname = args.shift
+            version = args.shift
+            data    = args.shift || {}
+
+            # this is gross but I'm not sure I actually care right now.
+            if (modname.is_a? String and [String, NilClass].include? version.class and data.is_a? Hash)
+              data[:boxen]   = :boxen
+              data[:version] = version
+              add_module(modname, data)
+            else
+              $logger.warn "#{@repo[:repo_name]}/Puppetfile: malformed boxen"
+            end
+          else
+            # Should we record unexpected Ruby code or just log it to stdout?
+            args = args.map {|a| a.is_a?(String) ? "'#{a}'" : a}.join(', ')
+            $logger.warn "#{@repo[:repo_name]}/Puppetfile: Unexpected invocation of #{name}(#{args})"
+          end
+        end
+
+        node.children.each do |n|
+          next unless n.is_a? RubyVM::AbstractSyntaxTree::Node
+
+          traverse(n)
+        end
+      rescue => e
+        puts e.message
+      end
+    end
+
+    def test()
+      require 'pry'
+      binding.pry
+    end
+  end
+end

data/lib/mvp/runner.rb

@@ -1,6 +1,10 @@
-require 'mvp/downloader'
-require 'mvp/uploader'
+require 'mvp/forge'
+require 'mvp/bigquery'
 require 'mvp/stats'
+require 'mvp/itemizer'
+require 'mvp/puppetfile_parser'
+
+require 'tty-spinner'
 
 class Mvp
   class Runner
@@ -11,52 +15,144 @@ class Mvp
     end
 
     def retrieve(target = :all, download = true)
-      downloader = Mvp::Downloader.new(@options)
+      bigquery = Mvp::Bigquery.new(@options)
 
-      [:authors, :modules, :releases].each do |thing|
-        next unless [:all, thing].include? target
-        downloader.retrieve(thing, download)
-      end
+      begin
+        [:authors, :modules, :releases, :validations].each do |thing|
+          next unless [:all, thing].include? target
+          spinner = mkspinner("Retrieving #{thing} ...")
+          data = bigquery.retrieve(thing)
+          save_json(thing, data)
+          spinner.success('(OK)')
+        end
 
-      if [:all, :validations].include? target
-        downloader.validations()
+      rescue => e
+        spinner.error("API error: #{e.message}")
+        $logger.error "API error: #{e.message}"
+        $logger.debug e.backtrace.join("\n")
+        sleep 10
       end
     end
 
-    def upload(target = :all)
-      uploader = Mvp::Uploader.new(@options)
+    def mirror(target = :all)
+      forge    = Mvp::Forge.new(@options)
+      bigquery = Mvp::Bigquery.new(@options)
+      itemizer = Mvp::Itemizer.new(@options)
+      pfparser = Mvp::PuppetfileParser.new(@options)
 
-      [:authors, :modules, :releases, :validations, :github_mirrors].each do |thing|
-        next unless [:all, thing].include? target
-        uploader.send(thing)
+      begin
+        [:authors, :modules, :releases].each do |thing|
+          next unless [:all, thing].include? target
+          spinner = mkspinner("Mirroring #{thing}...")
+          bigquery.truncate(thing)
+          forge.retrieve(thing) do |data, offset|
+            spinner.update(title: "Mirroring #{thing} [#{offset}]...")
+            bigquery.insert(thing, data)
+          end
+          spinner.success('(OK)')
+        end
+
+        if [:all, :validations].include? target
+          spinner = mkspinner("Mirroring validations...")
+          modules = bigquery.get(:modules, [:slug])
+          bigquery.truncate(:validations)
+          forge.retrieve_validations(modules) do |data, offset|
+            spinner.update(title: "Mirroring validations [#{offset}]...")
+            bigquery.insert(:validations, data)
+          end
+          spinner.success('(OK)')
+        end
+
+        if [:all, :itemizations].include? target
+          spinner = mkspinner("Itemizing modules...")
+          bigquery.unitemized.each do |mod|
+            spinner.update(title: "Itemizing [#{mod[:slug]}]...")
+            rows = itemizer.itemized(mod)
+            bigquery.delete(:itemized, :module, mod[:slug])
+            bigquery.insert(:itemized, rows)
+          end
+          spinner.success('(OK)')
+        end
+
+        if [:all, :mirrors, :tables].include? target
+          @options[:gcloud][:mirrors].each do |entity|
+            spinner = mkspinner("Mirroring #{entity[:type]} #{entity[:name]} to BigQuery...")
+            bigquery.mirror_table(entity)
+            spinner.success('(OK)')
+          end
+        end
+
+        if [:all, :puppetfiles].include? target
+          spinner = mkspinner("Analyzing Puppetfile module references...")
+          if pfparser.suitable?
+            pfparser.sources = bigquery.module_sources
+            bigquery.puppetfiles.each do |repo|
+              spinner.update(title: "Analyzing [#{repo[:repo_name]}/Puppetfile]...")
+              rows = pfparser.parse(repo)
+              bigquery.delete(:puppetfile_usage, :repo_name, repo[:repo_name], :github)
+              bigquery.insert(:puppetfile_usage, rows, :github)
+            end
+            spinner.success('(OK)')
+          else
+            spinner.error("(Not functional on Ruby #{RUBY_VERSION})")
+          end
+        end
+
+      rescue => e
+        spinner.error("API error: #{e.message}")
+        $logger.error "API error: #{e.message}"
+        $logger.debug e.backtrace.join("\n")
+        sleep 10
       end
     end
 
-    def mirror(target = :all)
-      downloader = Mvp::Downloader.new(@options)
-      uploader   = Mvp::Uploader.new(@options)
+    def analyze
+      bigquery = Mvp::Bigquery.new(@options)
+      itemizer = Mvp::Itemizer.new(@options)
 
-      # validations are downloaded with modules
-      [:authors, :modules, :releases].each do |thing|
-        next unless [:all, thing].include? target
-          uploader.truncate(thing)
-          downloader.mirror(thing, uploader)
-      end
+      begin
+        spinner = mkspinner("Analyzing modules...")
+        modules = bigquery.get(:modules, [:owner, :name, :version, :downloads])
+        modules = modules.sample(@options[:count]) if @options[:count]
+
+        require 'csv'
+        csv_string = CSV.generate do |csv|
+          modules.each do |mod|
+            spinner.stop if @options[:debug]
+            rows = itemizer.analyze(mod, @options[:script], @options[:debug])
+            spinner.start if @options[:debug]
+
+            next unless rows
+            spinner.update(title: mod[:name])
+            rows.each {|row| csv << row}
+          end
+        end
 
-      if [:all, :mirrors].include? target
-        uploader.github_mirrors()
+        File.write(@options[:output_file], csv_string)
+        spinner.success('(OK)')
       end
     end
 
     def stats(target)
       stats = Mvp::Stats.new(@options)
 
-      [:authors, :modules, :releases, :relationships, :github, :validations].each do |thing|
+      [:authors, :modules, :releases, :relationships, :validations].each do |thing|
         next unless [:all, thing].include? target
         stats.send(thing)
       end
     end
 
+    def mkspinner(title)
+      spinner = TTY::Spinner.new("[:spinner] :title")
+      spinner.update(title: title)
+      spinner.auto_spin
+      spinner
+    end
+
+    def save_json(thing, data)
+      File.write("#{@cachedir}/#{thing}.json", data.to_json)
+    end
+
     def test()
       require 'pry'
       binding.pry

data/lib/mvp/stats.rb

@@ -19,7 +19,8 @@ class Mvp
 
     def draw_graph(series, width, title = nil)
       series.compact!
-      graph    = []
+      width = [width, series.size].min
+      graph = []
       (bins, freqs) = series.histogram(:bin_width => width)
 
       bins.each_with_index do |item, index|
@@ -44,6 +45,20 @@ class Mvp
       days_ago(datestr)/365
     end
 
+    def current_releases
+      return @current_releases if @current_releases
+
+      data_m  = load('modules').reject {|m| m['owner'] == 'puppetlabs' }
+      data_r  = load('releases').reject {|m| m['owner'] == 'puppetlabs' }
+
+      @current_releases = data_m.map {|mod|
+        name = mod['slug']
+        curr = mod['releases'].first
+
+        data_r.find {|r| r['slug'] == "#{name}-#{curr}" }
+      }.compact
+    end
+
     def tally_author_info(releases, target, scope='module_count')
       # update the author records with the fields we need
       target.each do |author|
@@ -52,7 +67,7 @@ class Mvp
       end
 
       releases.each do |mod|
-        username = mod['module']['owner']['username']
+        username = mod['owner']
         score    = mod['validation_score']
         author   = target.select{|m| m['username'] == username}.first
 
@@ -111,9 +126,10 @@ class Mvp
     end
 
     def modules()
-      data_m  = load('modules').reject {|m| m['owner']['username'] == 'puppetlabs' }
+      data_m  = load('modules').reject {|m| m['owner'] == 'puppetlabs' }
       data_a  = load('authors').reject {|u| u['username'] == 'puppetlabs' or u['module_count'] == 0}
-      current = data_m.map {|m| m['current_release'] }
+
+      current = current_releases
 
       tally_author_info(current, data_a, 'module_count')
 
@@ -155,7 +171,7 @@ class Mvp
     end
 
     def releases()
-      data_r  = load('releases').reject {|m| m['module']['owner']['username'] == 'puppetlabs' }
+      data_r  = load('releases').reject {|m| m['owner'] == 'puppetlabs' }
       data_a  = load('authors').reject {|u| u['username'] == 'puppetlabs' or u['module_count'] == 0}
 
       tally_author_info(data_r, data_a, 'release_count')
@@ -236,12 +252,12 @@ class Mvp
     end
 
     def relationships()
-      data_m  = load('modules').reject {|m| m['owner']['username'] == 'puppetlabs' }
       data_a  = load('authors').reject {|u| u['username'] == 'puppetlabs' or u['module_count'] == 0}
-      current = data_m.map {|m| m['current_release'] }
+      current = current_releases.dup
 
       current.each do |mod|
-          mod['metadata']['dependants'] = []
+        mod['metadata'] = JSON.parse(mod['metadata'])
+        mod['metadata']['dependants'] = []
       end
       current.each do |mod|
         mod['metadata']['dependencies'].each do |dependency|
@@ -257,7 +273,7 @@ class Mvp
         count  = mod['metadata']['dependants'].count
         next unless count > 0
 
-        author = data_a.select{|m| m['username'] == mod['module']['owner']['username']}.first
+        author = data_a.select{|m| m['username'] == mod['owner']}.first
         author['dependants'] << count
       end
       data_a.each { |a| a['average_dependants'] = average(a['dependants']) }
@@ -280,6 +296,7 @@ class Mvp
                                         author['module_count'],
                                         author['release_count'] ]
       end
+      puts
     end
 
     def github()
@@ -328,7 +345,7 @@ class Mvp
     end
 
     def validations()
-      puts 'got nothing for you yet'
+      puts 'No validations yet'
     end
 
     def test()

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: puppet-community-mvp
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.7
 platform: ruby
 authors:
 - Ben Ford
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-10-26 00:00:00.000000000 Z
+date: 2021-08-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -109,7 +109,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: google-cloud
+  name: google-cloud-bigquery
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: puppet-itemize
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -137,13 +151,14 @@ files:
 - LICENSE
 - README.md
 - bin/mvp
+- bin/pftest.rb
 - lib/mvp.rb
-- lib/mvp/downloader.rb
+- lib/mvp/bigquery.rb
+- lib/mvp/forge.rb
 - lib/mvp/itemizer.rb
-- lib/mvp/monkeypatches.rb
+- lib/mvp/puppetfile_parser.rb
 - lib/mvp/runner.rb
 - lib/mvp/stats.rb
-- lib/mvp/uploader.rb
 homepage: 
 licenses:
 - Apache 2
@@ -163,8 +178,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.10
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Generate some stats about the Puppet Community.

data/lib/mvp/monkeypatches.rb

@@ -1,8 +0,0 @@
-# BigQuery uses newline delimited json
-# https://en.wikipedia.org/wiki/JSON_streaming#Line-delimited_JSON
-
-class Array
-  def to_newline_delimited_json
-    self.map(&:to_json).join("\n")
-  end
-end