puppet-community-mvp 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 8ed5308091443f5847159a6a481611fba281f4d5
-  data.tar.gz: f7bb0dd50ea248c04b5809144d67355ad7c6c202
+  metadata.gz: '082fa42831056dfcb39a9cf587c9128728ede8af'
+  data.tar.gz: c401376ae86adfc9ffcf2631cd607bcd1bbe8c6e
 SHA512:
-  metadata.gz: ececdc2a2121c4054fc49b16385892e78364b9ed197b2ac3e38a5542de2f5be94cf52ac9e6d6e1590c7e91b912fa54f4e9a70e71e60ae831fe545b98731021ee
-  data.tar.gz: 5f87defac101d2105403c0b5b54d34c37cac1b552fc77a2008d7d62624df41c8d1d4813be9c2cf15ded62942d6e0cbc70f36862d39866caa2bd77d6d9528aba0
+  metadata.gz: 396050b127e436e2c020836a051426a43c025ed3452983e42450f186ee1b04486c0efb7adfbd010a658a43865c4173cd6262ae16308363fd9847fa2969269f25
+  data.tar.gz: 35af39cff28f02378e7b7da71130c76a01cb06e4ec73b884b0ada051b3414c580e5fa578d606096d613940da70b3d3feb416ed5c23f77aa3238fbdaa894a9ca3

data/bin/mvp

@@ -8,7 +8,7 @@ require 'logger'
 require 'mvp'
 
 NAME     = File.basename($PROGRAM_NAME)
-options  = {:config => File.expand_path('~/.mvp.config.yaml')}
+options  = {:config => File.expand_path('~/.mvp/config.yaml')}
 optparse = OptionParser.new { |opts|
   opts.banner = "Usage : #{NAME} [command] [target] [options]
 
@@ -21,6 +21,9 @@ The following CLI commands are available.
   * upload | insert [target]
       * Uploads data to BigQuery
       * Optional targets: all, authors, modules, releases, mirrors
+  * mirror [target]
+      * Runs the download & then upload tasks.
+      * Optional targets: all, authors, modules, releases
   * stats
       * Print out a summary of interesting stats.
 "
@@ -80,6 +83,7 @@ options[:gcloud][:project] ||= 'puppet'
 options[:gcloud][:keyfile] ||= '~/.mvp/credentials.json'
 
 options[:cachedir]           = File.expand_path(options[:cachedir])
+options[:github_data]        = File.expand_path(options[:github_data])
 options[:gcloud][:keyfile]   = File.expand_path(options[:gcloud][:keyfile])
 FileUtils.mkdir_p(options[:cachedir])
 

data/lib/mvp/downloader.rb

@@ -7,15 +7,35 @@ require 'mvp/monkeypatches'
 class Mvp
   class Downloader
     def initialize(options = {})
-      @cachedir = options[:cachedir]
-      @forgeapi = options[:forgeapi] ||'https://forgeapi.puppet.com'
+      @useragent = 'Puppet Community Stats Monitor'
+      @cachedir  = options[:cachedir]
+      @forgeapi  = options[:forgeapi] ||'https://forgeapi.puppet.com'
+    end
+
+    def mirror(entity, uploader)
+      # using authors for git repo terminology consistency
+      item = (entity == :authors) ? 'users' : entity.to_s
+      download(item) do |data|
+        case entity
+        when :modules
+          uploader.insert(:validations, flatten_validations(retrieve_validations(data)))
+          data = flatten_modules(data)
+        when :releases
+          data = flatten_releases(data)
+        end
+
+        uploader.insert(entity, data)
+      end
     end
 
     def retrieve(entity, download = true)
       if download
         # I am focusing on authorship rather than just users, so for now I'm using the word authors
         item = (entity == :authors) ? 'users' : entity.to_s
-        data = download(item)
+        data = []
+        download(item) do |resp|
+          data.concat resp
+        end
         save_json(entity, data)
       else
         data = File.read("#{@cachedir}/#{entity}.json")
@@ -30,9 +50,35 @@ class Mvp
       save_nld_json(entity.to_s, data)
     end
 
-    def validations()
+    def retrieve_validations(modules, period = 25)
       results = {}
-      cache   = "#{@cachedir}/modules.json"
+
+      begin
+        offset   = 0
+        endpoint = "/private/validations/"
+        modules.each do |mod|
+          name = "#{mod['owner']['username']}-#{mod['name']}"
+          response = HTTParty.get("#{@forgeapi}#{endpoint}#{name}", headers: {'User-Agent' => @useragent})
+          raise "Forge Error: #{@response.body}" unless response.code == 200
+
+          results[name] = JSON.parse(response.body)
+          offset       += 1
+
+          if block_given? and (offset % period == 0)
+            yield offset
+            GC.start
+          end
+        end
+      rescue => e
+        $logger.error e.message
+        $logger.debug e.backtrace.join("\n")
+      end
+
+      results
+    end
+
+    def validations()
+      cache = "#{@cachedir}/modules.json"
 
       if File.exist? cache
         module_data = JSON.parse(File.read(cache))
@@ -41,22 +87,12 @@ class Mvp
       end
 
       begin
-        offset   = 0
-        endpoint = "/private/validations/"
-        spinner  = TTY::Spinner.new("[:spinner] :title")
+        spinner = TTY::Spinner.new("[:spinner] :title")
         spinner.update(title: "Downloading module validations ...")
         spinner.auto_spin
 
-        module_data.each do |mod|
-          name = "#{mod['owner']['username']}-#{mod['name']}"
-          response = HTTParty.get("#{@forgeapi}#{endpoint}#{name}", headers: {"User-Agent" => "Puppet Community Stats Monitor"})
-          raise "Forge Error: #{@response.body}" unless response.code == 200
-
-          data          = JSON.parse(response.body)
-          offset       += 1
-          results[name] = data
-
-          spinner.update(title: "Downloading module validations [#{offset}]...") if (offset % 25 == 0)
+        results = retrieve_validations(module_data) do |offset|
+          spinner.update(title: "Downloading module validations [#{offset}]...")
         end
 
         spinner.success('(OK)')
@@ -72,7 +108,7 @@ class Mvp
     end
 
     def download(entity)
-      results = []
+       raise 'Please process downloaded data by passing a block' unless block_given?
 
       begin
         offset   = 0
@@ -82,15 +118,19 @@ class Mvp
         spinner.auto_spin
 
         while endpoint do
-          response = HTTParty.get("#{@forgeapi}#{endpoint}", headers: {"User-Agent" => "Puppet Community Stats Monitor"})
+          response = HTTParty.get("#{@forgeapi}#{endpoint}", headers: {"User-Agent" => @useragent})
           raise "Forge Error: #{@response.body}" unless response.code == 200
-
           data = JSON.parse(response.body)
+
           offset  += 50
-          results += data['results']
           endpoint = data['pagination']['next']
 
-          spinner.update(title: "Downloading #{entity} [#{offset}]...") if (endpoint and (offset % 250 == 0))
+          yield munge_dates(data['results'])
+
+          if (endpoint and (offset % 250 == 0))
+            spinner.update(title: "Downloading #{entity} [#{offset}]...")
+            GC.start
+          end
         end
 
         spinner.success('(OK)')
@@ -100,7 +140,7 @@ class Mvp
         $logger.debug e.backtrace.join("\n")
       end
 
-      munge_dates(results)
+      nil
     end
 
     # transform dates into a format that bigquery knows
@@ -138,7 +178,7 @@ class Mvp
         row['source']            = row['current_release']['metadata']['source']
         row['project_page']      = row['current_release']['metadata']['project_page']
         row['issues_url']        = row['current_release']['metadata']['issues_url']
-        row['tasks']             = row['current_release']['tasks'].map{|task| task['name']}
+        row['tasks']             = row['current_release']['tasks'].map{|task| task['name']} rescue []
 
         row['release_count']     = row['releases'].count rescue 0
         row['releases']          = row['releases'].map{|r| r['version']} rescue []
@@ -152,12 +192,12 @@ class Mvp
     def flatten_releases(data)
       data.each do |row|
         row['name']              = row['module']['name']
-        row['owner']             = row['module']['username']
+        row['owner']             = row['module']['owner']['username']
         row['license']           = row['metadata']['license']
         row['source']            = row['metadata']['source']
         row['project_page']      = row['metadata']['project_page']
         row['issues_url']        = row['metadata']['issues_url']
-        row['tasks']             = row['tasks'].map{|task| task['name']}
+        row['tasks']             = row['tasks'].map{|task| task['name']} rescue []
 
         simplify_metadata(row, row['metadata'])
         row.delete('module')
@@ -176,8 +216,8 @@ class Mvp
     end
 
     def simplify_metadata(data, metadata)
-      data['operatingsystem']   = metadata['operatingsystem_support'].map{|i| i['operatingsystem']}                       rescue nil
-      data['dependencies']      = metadata['dependencies'].map{|i| i['name']}                                             rescue nil
+      data['operatingsystem']   = metadata['operatingsystem_support'].map{|i| i['operatingsystem']}                       rescue []
+      data['dependencies']      = metadata['dependencies'].map{|i| i['name'].sub('/', '-')}                               rescue []
       data['puppet_range']      = metadata['requirements'].select{|r| r['name'] == 'puppet'}.first['version_requirement'] rescue nil
       data['metadata']          = metadata.to_json
 

data/lib/mvp/runner.rb

@@ -26,15 +26,26 @@ class Mvp
     def upload(target = :all)
       uploader = Mvp::Uploader.new(@options)
 
-      [:authors, :modules, :releases, :validations, :mirrors].each do |thing|
+      [:authors, :modules, :releases, :validations, :github_mirrors].each do |thing|
         next unless [:all, thing].include? target
         uploader.send(thing)
       end
     end
 
     def mirror(target = :all)
-      retrieve(target)
-      upload(target)
+      downloader = Mvp::Downloader.new(@options)
+      uploader   = Mvp::Uploader.new(@options)
+
+      # validations are downloaded with modules
+      [:authors, :modules, :releases].each do |thing|
+        next unless [:all, thing].include? target
+          uploader.truncate(thing)
+          downloader.mirror(thing, uploader)
+      end
+
+      if [:all, :mirrors].include? target
+        uploader.github_mirrors()
+      end
     end
 
     def stats(target)

data/lib/mvp/uploader.rb

@@ -12,6 +12,139 @@ class Mvp
         :credentials => Google::Cloud::Bigquery::Credentials.new(options[:gcloud][:keyfile]),
       )
       @dataset = @bigquery.dataset(options[:gcloud][:dataset])
+
+      raise "\nThere is a problem with the gCloud configuration: \n #{JSON.pretty_generate(options)}" if @dataset.nil?
+    end
+
+    def truncate(entity)
+      begin
+        case entity
+        when :authors
+          @dataset.table('forge_authors').delete rescue nil
+          @dataset.create_table('forge_authors') do |table|
+            table.name        = 'Forge Authors'
+            table.description = 'A list of all authors (users) on the Forge'
+            table.schema do |s|
+              s.integer   "module_count",  mode: :required
+              s.integer   "release_count", mode: :required
+              s.timestamp "created_at",    mode: :required
+              s.string    "display_name",  mode: :required
+              s.string    "username",      mode: :required
+              s.timestamp "updated_at",    mode: :required
+              s.string    "gravatar_id",   mode: :required
+              s.string    "slug",          mode: :required
+              s.string    "uri",           mode: :required
+            end
+          end
+
+        when :modules
+          # both modules and validations
+          @dataset.table('forge_modules').delete rescue nil
+          @dataset.create_table('forge_modules') do |table|
+            table.name        = 'Forge Modules'
+            table.description = 'All modules and their metadata on the Forge'
+            table.schema do |s|
+              s.string    "name",             mode: :required
+              s.string    "owner",            mode: :required
+              s.string    "version",          mode: :required
+              s.string    "slug",             mode: :required
+              s.string    "uri",              mode: :required
+              s.timestamp "created_at",       mode: :required
+              s.timestamp "updated_at",       mode: :required
+              s.string    "tasks",            mode: :repeated
+              s.string    "homepage_url"
+              s.string    "project_page"
+              s.string    "issues_url"
+              s.string    "source"
+              s.boolean   "supported"
+              s.string    "endorsement"
+              s.string    "module_group"
+              s.boolean   "pdk"
+              s.string    "operatingsystem",  mode: :repeated
+              s.integer   "release_count",    mode: :required
+              s.integer   "downloads",        mode: :required
+              s.integer   "feedback_score"
+              s.integer   "validation_score"
+              s.string    "releases",         mode: :repeated
+              s.string    "puppet_range"
+              s.boolean   "puppet_2x"
+              s.boolean   "puppet_3x"
+              s.boolean   "puppet_4x"
+              s.boolean   "puppet_5x"
+              s.boolean   "puppet_6x"
+              s.string    "superseded_by"
+              s.string    "deprecated_for"
+              s.timestamp "deprecated_at"
+              s.timestamp "deleted_at"
+              s.string    "dependencies",     mode: :repeated
+              s.string    "license"
+              s.string    "metadata",         mode: :required
+            end
+          end
+
+          @dataset.table('forge_validations').delete rescue nil
+          @dataset.create_table('forge_validations') do |table|
+            table.name        = 'Forge Module Validations'
+            table.description = 'Validation scores for all the modules on the Forge'
+            table.schema do |s|
+              s.integer "total"
+              s.integer "parser"
+              s.integer "metadata"
+              s.integer "lint"
+              s.string  "name",     mode: :required
+            end
+          end
+
+        when :releases
+          @dataset.table('forge_releases').delete rescue nil
+          @dataset.create_table('forge_releases') do |table|
+            table.name        = 'Forge Releases'
+            table.description = 'Releases of all modules on the Forge'
+            table.schema do |s|
+              s.string    "name",             mode: :required
+              s.string    "owner",            mode: :required
+              s.string    "version",          mode: :required
+              s.string    "slug",             mode: :required
+              s.string    "uri",              mode: :required
+              s.timestamp "created_at",       mode: :required
+              s.timestamp "updated_at",       mode: :required
+              s.timestamp "deleted_at"
+              s.string    "deleted_for"
+              s.string    "tasks",            mode: :repeated
+              s.string    "project_page"
+              s.string    "issues_url"
+              s.string    "source"
+              s.boolean   "supported"
+              s.boolean   "pdk"
+              s.string    "tags",             mode: :repeated
+              s.string    "operatingsystem",  mode: :repeated
+              s.integer   "downloads",        mode: :required
+              s.integer   "feedback_score"
+              s.integer   "validation_score"
+              s.string    "puppet_range"
+              s.boolean   "puppet_2x"
+              s.boolean   "puppet_3x"
+              s.boolean   "puppet_4x"
+              s.boolean   "puppet_5x"
+              s.boolean   "puppet_6x"
+              s.string    "dependencies",     mode: :repeated
+              s.string    "file_uri",         mode: :required
+              s.string    "file_md5",         mode: :required
+              s.integer   "file_size",        mode: :required
+              s.string    "changelog"
+              s.string    "reference"
+              s.string    "readme"
+              s.string    "license"
+              s.string    "metadata",         mode: :required
+            end
+          end
+
+        end
+      rescue => e
+        $logger.error e.message
+        $logger.debug e.backtrace.join("\n")
+        @channels = @dataset.table('slack_channels')
+      end
     end
 
     def authors()
@@ -30,7 +163,7 @@ class Mvp
       upload('validations')
     end
 
-    def mirrors()
+    def github_mirrors()
       @mirrors.each do |entity|
         begin
           spinner = TTY::Spinner.new("[:spinner] :title")
@@ -62,6 +195,19 @@ class Mvp
       end
     end
 
+    def insert(entity, data)
+      table    = @dataset.table("forge_#{entity}")
+      response = table.insert(data)
+
+      unless response.success?
+        errors = {}
+        response.insert_errors.each do |err|
+          errors[err.row['slug']] = err.errors
+        end
+        $logger.error JSON.pretty_generate(errors)
+      end
+    end
+
     def upload(entity)
       begin
         spinner = TTY::Spinner.new("[:spinner] :title")

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: puppet-community-mvp
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Ben Ford
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-06-27 00:00:00.000000000 Z
+date: 2018-09-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -163,7 +163,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2.3
+rubygems_version: 2.6.10
 signing_key: 
 specification_version: 4
 summary: Generate some stats about the Puppet Community.