puppet-community-mvp 0.0.4 → 0.0.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: e45ff6e06d776fb8c3129789b4f469f25f86ea59
-  data.tar.gz: 640dd10a4620d77281ebb58ec4d097dc40b9bfbd
+  metadata.gz: 3f5ed8c2978cad58a0ce52346eb854b3b4e82a9a
+  data.tar.gz: a5637f505ed5fe8c74a22f9633723b9211ad3c4a
 SHA512:
-  metadata.gz: 00fffd365248a810b1542eb7e4e370398330d469834a2e5cecab70f0fe370149d8bc87ac7027c9260c6a2d0a8f24a06915e5946bf5d84c5dd6c07a8e4ac95ac3
-  data.tar.gz: ac1e153462e94226eb56621e687667f369aa0ca35f3d34dd6e3c9096e0d7dc56dcb1f4584f0b2072cda3f47a5c5f1b2b645f106d5f314ef45fdcc3d9f6af6a25
+  metadata.gz: 3ad31ba42a0e2f96ae8254bf383e523c74890fa219cb1b27946cae7718aba76b2d3532149dc98dfc1d59b5e47c4d31c70156c96203659425b55607c6a0d948d1
+  data.tar.gz: f8970bb1a709f807e3e5d2f45e2c74f7adbea034fc5d9293024e76f4dfb8b2e9eb59ecf636e38b12d69ce224867bbc96570f5e9ed1ddb5092783888e145ceec9

data/bin/mvp

@@ -13,16 +13,14 @@ optparse = OptionParser.new { |opts|
   opts.banner = "Usage : #{NAME} [command] [target] [options]
 
 This tool will scrape the Puppet Forge API for interesting module & author stats.
-The following CLI commands are available.
+It can also mirror public BigQuery tables or views into our dataset for efficiency,
+or download and itemize each Forge module.
 
-  * get | retrieve | download [target]
-      * Downloads and caches all Forge metadata.
-      * Optional targets: all, authors, modules, releases
-  * upload | insert [target]
-      * Uploads data to BigQuery
-      * Optional targets: all, authors, modules, releases, mirrors
   * mirror [target]
       * Runs the download & then upload tasks.
+      * Optional targets: all, authors, modules, releases, validations, itemizations, puppetfiles, tables
+  * get | retrieve | download [target]
+      * Downloads and caches data locally so you can run the stats task.
       * Optional targets: all, authors, modules, releases
   * stats
       * Print out a summary of interesting stats.
@@ -64,6 +62,10 @@ The following CLI commands are available.
     options[:debug] = true
   end
 
+  opts.on("-n", "--noop", "Don't actually upload data.") do
+    options[:noop] = true
+  end
+
   opts.separator('')
 
   opts.on("-h", "--help", "Displays this help") do
@@ -100,14 +102,6 @@ when 'get', 'retrieve', 'download'
   target ||= :all
   runner.retrieve(target.to_sym)
 
-when 'transform'
-  target ||= :all
-  runner.retrieve(target.to_sym, false)
-
-when 'insert', 'upload'
-  target ||= :all
-  runner.upload(target.to_sym)
-
 when 'mirror'
   target ||= :all
   runner.mirror(target.to_sym)

data/bin/pftest.rb

@@ -0,0 +1,22 @@
+#! /usr/bin/env ruby
+
+require 'mvp/puppetfile_parser'
+require 'open-uri'
+require 'json'
+require 'logger'
+
+$logger           = Logger::new(STDOUT)
+$logger.level     = Logger::INFO
+$logger.formatter = proc { |severity,datetime,progname,msg| "#{severity}: #{msg}\n" }
+
+pf = open(ARGV.first)
+parser = Mvp::PuppetfileParser.new()
+
+
+repo = {
+  :repo_name => 'testing',
+  :md5       => 'wakka wakka',
+  :content   => pf.read,
+}
+
+puts JSON.pretty_generate(parser.parse(repo))

data/lib/mvp.rb

@@ -1,4 +1,2 @@
 require 'mvp/runner'
-require 'mvp/downloader'
-require 'mvp/uploader'
-require 'mvp/stats'
+require 'mvp/stats'

data/lib/mvp/{uploader.rb → bigquery.rb}

@@ -3,10 +3,10 @@ require 'tty-spinner'
 require "google/cloud/bigquery"
 
 class Mvp
-  class Uploader
+  class Bigquery
     def initialize(options = {})
+      @options  = options
       @cachedir = options[:cachedir]
-      @mirrors  = options[:gcloud][:mirrors]
       @bigquery = Google::Cloud::Bigquery.new(
         :project_id  => options[:gcloud][:project],
         :credentials => Google::Cloud::Bigquery::Credentials.new(options[:gcloud][:keyfile]),
@@ -27,9 +27,24 @@ class Mvp
                                                           s.integer "count",   mode: :required
                                                         end
                                                       end
+
+      @puppetfile_usage = @dataset.table('github_puppetfile_usage') || @dataset.create_table('github_puppetfile_usage') do |table|
+                                                                          table.name        = 'Puppetfile Module Usage'
+                                                                          table.description = 'A list of all modules referenced in public Puppetfiles'
+                                                                          table.schema do |s|
+                                                                            s.string    "repo_name", mode: :required
+                                                                            s.string    "module",    mode: :required
+                                                                            s.string    "type",      mode: :required
+                                                                            s.string    "source"
+                                                                            s.string    "version"
+                                                                            s.string    "md5",       mode: :required
+                                                                          end
+                                                                        end
     end
 
     def truncate(entity)
+      return if @options[:noop]
+
       begin
         case entity
         when :authors
@@ -163,95 +178,85 @@ class Mvp
       end
     end
 
-    def authors()
-      upload('authors')
-    end
-
-    def modules()
-      upload('modules')
+    def retrieve(entity)
+      get(entity, ['*'])
     end
 
-    def releases()
-      upload('releases')
-    end
+    def mirror_table(entity)
+      return if @options[:noop]
 
-    def validations()
-      upload('validations')
-    end
-
-    def github_mirrors()
-      @mirrors.each do |entity|
-        begin
-          spinner = TTY::Spinner.new("[:spinner] :title")
-          spinner.update(title: "Mirroring #{entity[:type]} #{entity[:name]} to BigQuery...")
-          spinner.auto_spin
-
-          case entity[:type]
-          when :view
-            @dataset.table(entity[:name]).delete rescue nil # delete if exists
-            @dataset.create_view(entity[:name], entity[:query],
-                                  :legacy_sql => true)
-
-          when :table
-            job = @dataset.query_job(entity[:query],
-                                  :legacy_sql => true,
-                                  :write      => 'truncate',
-                                  :table      => @dataset.table(entity[:name], :skip_lookup => true))
-            job.wait_until_done!
+      begin
+        case entity[:type]
+        when :view
+          @dataset.table(entity[:name]).delete rescue nil # delete if exists
+          @dataset.create_view(entity[:name], entity[:query])
 
-          else
-            $logger.error "Unknown mirror type: #{entity[:type]}"
-          end
+        when :table
+          job = @dataset.query_job(entity[:query],
+                                :write      => 'truncate',
+                                :table      => @dataset.table(entity[:name], :skip_lookup => true))
+          job.wait_until_done!
 
-          spinner.success('(OK)')
-        rescue => e
-          spinner.error("(Google Cloud error: #{e.message})")
-          $logger.error e.backtrace.join("\n")
+        else
+          $logger.error "Unknown mirror type: #{entity[:type]}"
         end
+      rescue => e
+        $logger.error("(Google Cloud error: #{e.message})")
+        $logger.debug e.backtrace.join("\n")
       end
     end
 
-    def insert(entity, data)
-      table    = @dataset.table("forge_#{entity}")
+    def insert(entity, data, suite = 'forge')
+      return if @options[:noop]
+      return if data.empty?
+
+      table    = @dataset.table("#{suite}_#{entity}")
       response = table.insert(data)
 
       unless response.success?
-        errors = {}
         response.insert_errors.each do |err|
-          errors[err.row['slug']] = err.errors
+          $logger.error JSON.pretty_generate(err.row)
+          $logger.error JSON.pretty_generate(err.errors)
         end
-        $logger.error JSON.pretty_generate(errors)
       end
     end
 
-    def upload(entity)
-      begin
-        spinner = TTY::Spinner.new("[:spinner] :title")
-        spinner.update(title: "Uploading #{entity} to BigQuery ...")
-        spinner.auto_spin
+    def delete(entity, field, match, suite = 'forge')
+      @dataset.query("DELETE FROM #{suite}_#{entity} WHERE #{field} = '#{match}'")
+    end
 
-        @dataset.load("forge_#{entity}", "#{@cachedir}/nld_#{entity}.json",
-                        :write      => 'truncate',
-                        :autodetect => true)
+    def get(entity, fields, suite = 'forge')
+      raise 'pass fields as an array' unless fields.is_a? Array
+      @dataset.query("SELECT #{fields.join(', ')} FROM #{suite}_#{entity}")
+    end
 
-#         table = @dataset.table("forge_#{entity}")
-#         File.readlines("#{@cachedir}/nld_#{entity}.json").each do |line|
-#           data = JSON.parse(line)
-#
-#           begin
-#             table.insert data
-#           rescue
-#             require 'pry'
-#             binding.pry
-#           end
-#         end
+    def module_sources()
+      get('modules', ['slug', 'source'])
+    end
 
+    def puppetfiles()
+      sql = 'SELECT f.repo_name, f.path, c.content, c.md5
+                FROM github_puppetfile_files AS f
+                JOIN github_puppetfile_contents AS c
+                  ON c.id = f.id
 
-        spinner.success('(OK)')
-      rescue => e
-        spinner.error("(Google Cloud error: #{e.message})")
-        $logger.error e.backtrace.join("\n")
-      end
+              WHERE c.md5 NOT IN (
+                SELECT u.md5
+                FROM github_puppetfile_usage AS u
+                WHERE u.repo_name = f.repo_name
+              ) AND LOWER(repo_name) NOT LIKE "%boxen%"'
+      @dataset.query(sql)
+    end
+
+    def unitemized()
+      sql = 'SELECT m.name, m.slug, m.version, m.dependencies
+              FROM forge_modules AS m
+              WHERE m.version NOT IN (
+                SELECT i.version
+                FROM forge_itemized AS i
+                WHERE module = m.slug
+              )'
+      @dataset.query(sql)
     end
 
     def version_itemized?(mod, version)

data/lib/mvp/{downloader.rb → forge.rb}

@@ -2,151 +2,82 @@ require 'json'
 require 'httparty'
 require 'tty-spinner'
 require 'semantic_puppet'
-require 'mvp/monkeypatches'
-require 'mvp/itemizer'
 
 class Mvp
-  class Downloader
+  class Forge
     def initialize(options = {})
       @useragent = 'Puppet Community Stats Monitor'
-      @cachedir  = options[:cachedir]
       @forgeapi  = options[:forgeapi] ||'https://forgeapi.puppet.com'
-      @itemizer  = Mvp::Itemizer.new(options)
     end
 
-    def mirror(entity, uploader)
-      # using authors for git repo terminology consistency
-      item = (entity == :authors) ? 'users' : entity.to_s
-      download(item) do |data|
-        case entity
-        when :modules
-          uploader.insert(:validations, flatten_validations(retrieve_validations(data)))
-          data = flatten_modules(data)
-
-          @itemizer.run!(data, uploader)
-        when :releases
-          data = flatten_releases(data)
-        end
-
-        uploader.insert(entity, data)
-      end
-    end
-
-    def retrieve(entity, download = true)
-      if download
-        # I am focusing on authorship rather than just users, so for now I'm using the word authors
-        item = (entity == :authors) ? 'users' : entity.to_s
-        data = []
-        download(item) do |resp|
-          data.concat resp
-        end
-        save_json(entity, data)
-      else
-        data = File.read("#{@cachedir}/#{entity}.json")
-      end
-
-      case entity
-      when :modules
-        data = flatten_modules(data)
-      when :releases
-        data = flatten_releases(data)
-      end
-      save_nld_json(entity.to_s, data)
-    end
-
-    def retrieve_validations(modules, period = 25)
-      results = {}
+    def retrieve(entity)
+      raise 'Please process downloaded data by passing a block' unless block_given?
 
+      # using authors for git repo terminology consistency
+      entity = :users if entity == :authors
       begin
         offset   = 0
-        endpoint = "/private/validations/"
-        modules.each do |mod|
-          name = "#{mod['owner']['username']}-#{mod['name']}"
-          response = HTTParty.get("#{@forgeapi}#{endpoint}#{name}", headers: {'User-Agent' => @useragent})
+        endpoint = "/v3/#{entity}?sort_by=downloads&limit=50"
+
+        while endpoint do
+          response = HTTParty.get("#{@forgeapi}#{endpoint}", headers: {"User-Agent" => @useragent})
           raise "Forge Error: #{@response.body}" unless response.code == 200
+          data    = JSON.parse(response.body)
+          results = munge_dates(data['results'])
+
+          case entity
+          when :modules
+            results = flatten_modules(results)
+          when :releases
+            results = flatten_releases(results)
+          end
 
-          results[name] = JSON.parse(response.body)
-          offset       += 1
+          yield results, offset
 
-          if block_given? and (offset % period == 0)
-            yield offset
+          offset  += 50
+          endpoint = data['pagination']['next']
+          if (endpoint and (offset % 250 == 0))
             GC.start
           end
         end
+
       rescue => e
         $logger.error e.message
         $logger.debug e.backtrace.join("\n")
       end
 
-      results
+      nil
     end
 
-    def validations()
-      cache = "#{@cachedir}/modules.json"
-
-      if File.exist? cache
-        module_data = JSON.parse(File.read(cache))
-      else
-        module_data = retrieve(:modules)
-      end
+    def retrieve_validations(modules, period = 25)
+      raise 'Please process validations by passing a block' unless block_given?
 
+      offset = 0
       begin
-        spinner = TTY::Spinner.new("[:spinner] :title")
-        spinner.update(title: "Downloading module validations ...")
-        spinner.auto_spin
+        modules.each_slice(period) do |group|
+          offset += period
+          results = group.map { |mod| validations(mod[:slug]) }
 
-        results = retrieve_validations(module_data) do |offset|
-          spinner.update(title: "Downloading module validations [#{offset}]...")
+          yield results, offset
+          GC.start
         end
-
-        spinner.success('(OK)')
       rescue => e
-        spinner.error('API error')
         $logger.error e.message
         $logger.debug e.backtrace.join("\n")
       end
 
-      save_json('validations', results)
-      save_nld_json('validations', flatten_validations(results))
-      results
+      nil
     end
 
-    def download(entity)
-       raise 'Please process downloaded data by passing a block' unless block_given?
+    def validations(name)
+      endpoint = "/private/validations/"
+      response = HTTParty.get("#{@forgeapi}#{endpoint}#{name}", headers: {'User-Agent' => @useragent})
+      raise "Forge Error: #{@response.body}" unless response.code == 200
 
-      begin
-        offset   = 0
-        endpoint = "/v3/#{entity}?sort_by=downloads&limit=50"
-        spinner  = TTY::Spinner.new("[:spinner] :title")
-        spinner.update(title: "Downloading #{entity} ...")
-        spinner.auto_spin
-
-        while endpoint do
-          response = HTTParty.get("#{@forgeapi}#{endpoint}", headers: {"User-Agent" => @useragent})
-          raise "Forge Error: #{@response.body}" unless response.code == 200
-          data = JSON.parse(response.body)
-
-          offset  += 50
-          endpoint = data['pagination']['next']
-
-          yield munge_dates(data['results'])
-
-          if (endpoint and (offset % 250 == 0))
-            spinner.update(title: "Downloading #{entity} [#{offset}]...")
-            GC.start
-          end
-        end
-
-        spinner.success('(OK)')
-      rescue => e
-        spinner.error('API error')
-        $logger.error e.message
-        $logger.debug e.backtrace.join("\n")
-      end
-
-      nil
+      flatten_validations(name, JSON.parse(response.body))
     end
 
+
     # transform dates into a format that bigquery knows
     def munge_dates(object)
       ["created_at", "updated_at", "deprecated_at", "deleted_at"].each do |field|
@@ -160,16 +91,6 @@ class Mvp
       object
     end
 
-    def save_json(thing, data)
-      File.write("#{@cachedir}/#{thing}.json", data.to_json)
-    end
-
-    # store data in a way that bigquery can grok
-    # uploading files is far easier than streaming data, when replacing a dataset
-    def save_nld_json(thing, data)
-      File.write("#{@cachedir}/nld_#{thing}.json", data.to_newline_delimited_json)
-    end
-
     def flatten_modules(data)
       data.each do |row|
         row['owner']             = row['owner']['username']
@@ -209,14 +130,12 @@ class Mvp
       data
     end
 
-    def flatten_validations(data)
-      data.map do |name, scores|
-        row = { 'name' => name }
-        scores.each do |entry|
-          row[entry['name']] = entry['score']
-        end
-        row
+    def flatten_validations(name, scores)
+      row = { 'name' => name }
+      scores.each do |entry|
+        row[entry['name']] = entry['score']
       end
+      row
     end
 
     def simplify_metadata(data, metadata)

data/lib/mvp/itemizer.rb

@@ -27,6 +27,14 @@ class Mvp
       end
     end
 
+    def itemized(mod)
+      modname = mod[:slug]
+      version = mod[:version]
+      baserow = { :module => modname, :version => version, :kind => 'admin', :element => 'version', :count => 0}
+
+      table(itemize(modname, version), mod) << baserow
+    end
+
     def download(path, modname, version)
       filename = "#{modname}-#{version}.tar.gz"
       Dir.chdir(path) do
@@ -58,10 +66,10 @@ class Mvp
     # Build a table with this schema
     # module | version | source | kind | element | count
     def table(itemized, data)
-      modname      = data['name']
-      slug         = data['slug']
-      version      = data['version']
-      dependencies = data['dependencies']
+      modname      = data[:name]
+      slug         = data[:slug]
+      version      = data[:version]
+      dependencies = data[:dependencies]
 
       itemized.map do |kind, elements|
         # the kind of element comes pluralized from puppet-itemize

data/lib/mvp/puppetfile_parser.rb

@@ -0,0 +1,171 @@
+class Mvp
+  class PuppetfileParser
+    def initialize(options = {})
+      @sources = {}
+      @modules = []
+      @repo    = nil
+    end
+
+    def suitable?
+      defined?(RubyVM::AbstractSyntaxTree)
+    end
+
+    def sources=(modules)
+      modules.each do |row|
+        next unless row[:source]
+        next if row[:source] == 'UNKNOWN'
+
+        @sources[canonical_git_repo(row[:source])] = row[:slug]
+      end
+    end
+
+    def parse(repo)
+      # This only works on Ruby 2.6+
+      return unless suitable?
+
+      begin
+        root = RubyVM::AbstractSyntaxTree.parse(repo[:content])
+      rescue SyntaxError => e
+        $logger.warn "Syntax error in #{repo[:repo_name]}/Puppetfile"
+        $logger.warn e.message
+      end
+
+      @repo    = repo
+      @modules = []
+      traverse(root)
+      @modules.compact.map do |row|
+        row[:repo_name] = repo[:repo_name]
+        row[:md5]       = repo[:md5]
+        row[:module]    = canonical_name(row[:module], row[:source])
+        stringify(row)
+      end
+    end
+
+    def stringify(row)
+      row.each do |key, value|
+        if value.is_a? RubyVM::AbstractSyntaxTree::Node
+          row[key] = :'#<programmatically generated via ruby code>'
+        end
+      end
+    end
+
+    def canonical_name(name, repo)
+      return name if name.include?('-')
+      repo = canonical_git_repo(repo)
+
+      return @sources[repo] if @sources.include?(repo)
+      name
+    end
+
+    def canonical_git_repo(repo)
+      return unless repo
+      return unless repo.is_a? String
+      repo.sub(/^git@github.com\:/, 'github.com/')
+          .sub(/^(git|https?)\:\/\//, '')
+          .sub(/\.git$/, '')
+    end
+
+    def add_module(name, args)
+      unless name.is_a? String
+        $logger.warn "Non string module name in #{@repo[:repo_name]}/Puppetfile"
+        return nil
+      end
+      name.gsub!('/', '-')
+      case args
+      when String, Symbol, NilClass
+        @modules << {
+          :module  => name,
+          :type    => :forge,
+          :source  => :forge,
+          :version => args,
+        }
+      when Hash
+        @modules << parse_args(name, args)
+      else
+        $logger.warn "#{@repo[:repo_name]}/Puppetfile: Unknown format: mod('#{name}', #{args.inspect})"
+      end
+    end
+
+    def parse_args(name, args)
+      data = {:module => name}
+
+      if args.include? :git
+        data[:type]    = :git
+        data[:source]  = args[:git]
+        data[:version] = args[:ref] || args[:tag] || args[:commit] || args[:branch] || :latest
+      elsif args.include? :svn
+        data[:type]    = :svn
+        data[:source]  = args[:svn]
+        data[:version] = args[:rev] || args[:revision] || :latest
+      elsif args.include? :boxen
+        data[:type]    = :boxen
+        data[:source]  = args[:repo]
+        data[:version] = args[:version] || :latest
+      else
+        $logger.warn "#{@repo[:repo_name]}/Puppetfile: Unknown args format: mod('#{name}', #{args.inspect})"
+        return nil
+      end
+
+      data
+    end
+
+    def traverse(node)
+      begin
+        if node.type == :FCALL
+          name = node.children.first
+          args = node.children.last.children.map do |item|
+            next if item.nil?
+
+            case item.type
+            when :HASH
+              Hash[*item.children.first.children.compact.map {|n| n.children.first }]
+            else
+              item.children.first
+            end
+          end.compact
+
+          case name
+          when :mod
+            add_module(args.shift, args.shift)
+          when :forge
+            # noop
+          when :moduledir
+            # noop
+          when :github
+            # oh boxen, you so silly.
+            # The order of the unpacking below *is* important.
+            modname = args.shift
+            version = args.shift
+            data    = args.shift || {}
+
+            # this is gross but I'm not sure I actually care right now.
+            if (modname.is_a? String and [String, NilClass].include? version.class and data.is_a? Hash)
+              data[:boxen]   = :boxen
+              data[:version] = version
+              add_module(modname, data)
+            else
+              $logger.warn "#{@repo[:repo_name]}/Puppetfile: malformed boxen"
+            end
+          else
+            # Should we record unexpected Ruby code or just log it to stdout?
+            args = args.map {|a| a.is_a?(String) ? "'#{a}'" : a}.join(', ')
+            $logger.warn "#{@repo[:repo_name]}/Puppetfile: Unexpected invocation of #{name}(#{args})"
+          end
+        end
+
+        node.children.each do |n|
+          next unless n.is_a? RubyVM::AbstractSyntaxTree::Node
+
+          traverse(n)
+        end
+      rescue => e
+        puts e.message
+      end
+    end
+
+    def test()
+      require 'pry'
+      binding.pry
+    end
+  end
+end

data/lib/mvp/runner.rb

@@ -1,6 +1,10 @@
-require 'mvp/downloader'
-require 'mvp/uploader'
+require 'mvp/forge'
+require 'mvp/bigquery'
 require 'mvp/stats'
+require 'mvp/itemizer'
+require 'mvp/puppetfile_parser'
+
+require 'tty-spinner'
 
 class Mvp
   class Runner
@@ -11,40 +15,94 @@ class Mvp
     end
 
     def retrieve(target = :all, download = true)
-      downloader = Mvp::Downloader.new(@options)
+      bigquery = Mvp::Bigquery.new(@options)
 
-      [:authors, :modules, :releases].each do |thing|
-        next unless [:all, thing].include? target
-        downloader.retrieve(thing, download)
-      end
+      begin
+        [:authors, :modules, :releases, :validations].each do |thing|
+          next unless [:all, thing].include? target
+          spinner = mkspinner("Retrieving #{thing} ...")
+          data = bigquery.retrieve(thing)
+          save_json(thing, data)
+          spinner.success('(OK)')
+        end
 
-      if [:all, :validations].include? target
-        downloader.validations()
+      rescue => e
+        spinner.error("API error: #{e.message}")
+        $logger.error "API error: #{e.message}"
+        $logger.debug e.backtrace.join("\n")
+        sleep 10
       end
     end
 
-    def upload(target = :all)
-      uploader = Mvp::Uploader.new(@options)
+    def mirror(target = :all)
+      forge    = Mvp::Forge.new(@options)
+      bigquery = Mvp::Bigquery.new(@options)
+      itemizer = Mvp::Itemizer.new(@options)
+      pfparser = Mvp::PuppetfileParser.new(@options)
 
-      [:authors, :modules, :releases, :validations, :github_mirrors].each do |thing|
-        next unless [:all, thing].include? target
-        uploader.send(thing)
-      end
-    end
+      begin
+        [:authors, :modules, :releases].each do |thing|
+          next unless [:all, thing].include? target
+          spinner = mkspinner("Mirroring #{thing}...")
+          bigquery.truncate(thing)
+          forge.retrieve(thing) do |data, offset|
+            spinner.update(title: "Mirroring #{thing} [#{offset}]...")
+            bigquery.insert(thing, data)
+          end
+          spinner.success('(OK)')
+        end
 
-    def mirror(target = :all)
-      downloader = Mvp::Downloader.new(@options)
-      uploader   = Mvp::Uploader.new(@options)
+        if [:all, :validations].include? target
+          spinner = mkspinner("Mirroring validations...")
+          modules = bigquery.get(:modules, [:slug])
+          bigquery.truncate(:validations)
+          forge.retrieve_validations(modules) do |data, offset|
+            spinner.update(title: "Mirroring validations [#{offset}]...")
+            bigquery.insert(:validations, data)
+          end
+          spinner.success('(OK)')
+        end
 
-      # validations are downloaded with modules
-      [:authors, :modules, :releases].each do |thing|
-        next unless [:all, thing].include? target
-          uploader.truncate(thing)
-          downloader.mirror(thing, uploader)
-      end
+        if [:all, :itemizations].include? target
+          spinner = mkspinner("Itemizing modules...")
+          bigquery.unitemized.each do |mod|
+            spinner.update(title: "Itemizing [#{mod[:slug]}]...")
+            rows = itemizer.itemized(mod)
+            bigquery.delete(:itemized, :module, mod[:slug])
+            bigquery.insert(:itemized, rows)
+          end
+          spinner.success('(OK)')
+        end
+
+        if [:all, :mirrors, :tables].include? target
+          @options[:gcloud][:mirrors].each do |entity|
+            spinner = mkspinner("Mirroring #{entity[:type]} #{entity[:name]} to BigQuery...")
+            bigquery.mirror_table(entity)
+            spinner.success('(OK)')
+          end
+        end
+
+        if [:all, :puppetfiles].include? target
+          spinner = mkspinner("Analyzing Puppetfile module references...")
+          if pfparser.suitable?
+            pfparser.sources = bigquery.module_sources
+            bigquery.puppetfiles.each do |repo|
+              spinner.update(title: "Analyzing [#{repo[:repo_name]}/Puppetfile]...")
+              rows = pfparser.parse(repo)
+              bigquery.delete(:puppetfile_usage, :repo_name, repo[:repo_name], :github)
+              bigquery.insert(:puppetfile_usage, rows, :github)
+            end
+            spinner.success('(OK)')
+          else
+            spinner.error("(Not functional on Ruby #{RUBY_VERSION})")
+          end
+        end
 
-      if [:all, :mirrors].include? target
-        uploader.github_mirrors()
+      rescue => e
+        spinner.error("API error: #{e.message}")
+        $logger.error "API error: #{e.message}"
+        $logger.debug e.backtrace.join("\n")
+        sleep 10
       end
     end
 
@@ -57,6 +115,17 @@ class Mvp
       end
     end
 
+    def mkspinner(title)
+      spinner = TTY::Spinner.new("[:spinner] :title")
+      spinner.update(title: title)
+      spinner.auto_spin
+      spinner
+    end
+
+    def save_json(thing, data)
+      File.write("#{@cachedir}/#{thing}.json", data.to_json)
+    end
+
     def test()
       require 'pry'
       binding.pry

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: puppet-community-mvp
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.0.5
 platform: ruby
 authors:
 - Ben Ford
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-10-31 00:00:00.000000000 Z
+date: 2019-03-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -151,13 +151,14 @@ files:
 - LICENSE
 - README.md
 - bin/mvp
+- bin/pftest.rb
 - lib/mvp.rb
-- lib/mvp/downloader.rb
+- lib/mvp/bigquery.rb
+- lib/mvp/forge.rb
 - lib/mvp/itemizer.rb
-- lib/mvp/monkeypatches.rb
+- lib/mvp/puppetfile_parser.rb
 - lib/mvp/runner.rb
 - lib/mvp/stats.rb
-- lib/mvp/uploader.rb
 homepage: 
 licenses:
 - Apache 2

data/lib/mvp/monkeypatches.rb

@@ -1,8 +0,0 @@
-# BigQuery uses newline delimited json
-# https://en.wikipedia.org/wiki/JSON_streaming#Line-delimited_JSON
-
-class Array
-  def to_newline_delimited_json
-    self.map(&:to_json).join("\n")
-  end
-end