puppet-community-mvp 0.0.2 → 0.0.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: '082fa42831056dfcb39a9cf587c9128728ede8af'
-  data.tar.gz: c401376ae86adfc9ffcf2631cd607bcd1bbe8c6e
+  metadata.gz: f77ade2721786fdca7fd96827ca510e0404a02a8
+  data.tar.gz: ad32f5a43392ed7f23d49d60db3fbe308ae73e2d
 SHA512:
-  metadata.gz: 396050b127e436e2c020836a051426a43c025ed3452983e42450f186ee1b04486c0efb7adfbd010a658a43865c4173cd6262ae16308363fd9847fa2969269f25
-  data.tar.gz: 35af39cff28f02378e7b7da71130c76a01cb06e4ec73b884b0ada051b3414c580e5fa578d606096d613940da70b3d3feb416ed5c23f77aa3238fbdaa894a9ca3
+  metadata.gz: be5282a77000b433c3aedd58ddfd29ea594df089007e38e74cb8b18c5c6b576ff048b218ec30cb9f60b8291faa1b62525f8d2c79694444bb3f95df2ede98efc0
+  data.tar.gz: 94894492ffba9d187f9178a17d51e412d80dc7b3434681bf43da79f422c6bbb0ddddba7f347b49ed048e0134a72eefe5ff3d83a615d5c4402d809f3e8010c1ac

data/bin/mvp

@@ -76,6 +76,7 @@ optparse.parse!
 options = (YAML.load_file(options[:config]) rescue {}).merge(options)
 
 options[:cachedir]         ||= '~/.mvp/cache'
+options[:github_data]      ||= '~/.mvp/github_modules.csv'
 options[:forgeapi]         ||= 'https://forgeapi.puppet.com'
 options[:gcloud]           ||= {}
 options[:gcloud][:dataset] ||= 'community'

data/lib/mvp/downloader.rb

@@ -3,6 +3,7 @@ require 'httparty'
 require 'tty-spinner'
 require 'semantic_puppet'
 require 'mvp/monkeypatches'
+require 'mvp/itemizer'
 
 class Mvp
   class Downloader
@@ -10,6 +11,7 @@ class Mvp
       @useragent = 'Puppet Community Stats Monitor'
       @cachedir  = options[:cachedir]
       @forgeapi  = options[:forgeapi] ||'https://forgeapi.puppet.com'
+      @itemizer  = Mvp::Itemizer.new(options)
     end
 
     def mirror(entity, uploader)
@@ -20,6 +22,8 @@ class Mvp
         when :modules
           uploader.insert(:validations, flatten_validations(retrieve_validations(data)))
           data = flatten_modules(data)
+
+          @itemizer.run!(data, uploader)
         when :releases
           data = flatten_releases(data)
         end
@@ -228,6 +232,7 @@ class Mvp
         data['puppet_4x']       = range.include? SemanticPuppet::Version.parse('4.99.99')
         data['puppet_5x']       = range.include? SemanticPuppet::Version.parse('5.99.99')
         data['puppet_6x']       = range.include? SemanticPuppet::Version.parse('6.99.99')
+        data['puppet_99x']      = range.include? SemanticPuppet::Version.parse('99.99.99')  # identify unbounded ranges
       end
     end
 

data/lib/mvp/itemizer.rb

@@ -0,0 +1,84 @@
+require 'httparty'
+require 'fileutils'
+require 'puppet_x/binford2k/itemize'
+require 'puppet_x/binford2k/itemize/runner'
+
+class Mvp
+  class Itemizer
+    def initialize(options = {})
+      @useragent = 'Puppet Community Stats Monitor'
+      @forge     = options[:forge] ||'https://forge.puppet.com'
+    end
+
+    def run!(data, uploader)
+      data.each do |mod|
+        modname = mod['slug']
+        version = mod['version']
+        return if uploader.version_itemized?(modname, version)
+
+        $logger.debug "Itemizing #{modname}-#{version}"
+        begin
+          rows = table(itemize(modname, version), mod)
+          uploader.insert(:itemized, rows) unless rows.empty?
+        rescue => e
+          $logger.error e.message
+          $logger.debug e.backtrace.join("\n")
+        end
+      end
+    end
+
+    def download(path, modname, version)
+      filename = "#{modname}-#{version}.tar.gz"
+      Dir.chdir(path) do
+        File.open(filename, "w") do |file|
+          file << HTTParty.get( "#{@forge}/v3/files/#{filename}" )
+        end
+        system("tar -xf #{filename}")
+        FileUtils.rm(filename)
+      end
+    end
+
+    def itemize(modname, version)
+      Dir.mktmpdir('mvp') do |path|
+        download(path, modname, version)
+
+        # not all modules have manifests
+        manifests = "#{path}/#{modname}-#{version}/manifests"
+        next {} unless File.directory?(manifests)
+
+        options = {
+          :manifests => [manifests],
+          :external  => true,
+        }
+        runner = Puppet_X::Binford2k::Itemize::Runner.new(options).run!
+        runner.results
+      end
+    end
+
+    # Build a table with this schema
+    # module | version | source | kind | element | count
+    def table(itemized, data)
+      modname      = data['slug']
+      version      = data['version']
+      dependencies = data['dependencies']
+
+      itemized.map do |kind, elements|
+        # the kind of element comes pluralized from puppet-itemize
+        kind = kind.to_s
+        kind = kind.end_with?('ses') ? kind.chomp('es') : kind.chomp('s')
+        elements.map do |name, count|
+          # TODO: this may suffer from collisions, (module foo, function foo, for example)
+          depname = name.split('::').first
+          source  = dependencies.find {|row| row.split('-').last == depname} rescue nil
+
+          { :module => modname, :version => version, :source => source, :kind => kind, :element => name, :count => count }
+        end
+      end.flatten(1)
+    end
+
+    def test()
+      require 'pry'
+      binding.pry
+    end
+  end
+end

data/lib/mvp/uploader.rb

@@ -14,6 +14,19 @@ class Mvp
       @dataset = @bigquery.dataset(options[:gcloud][:dataset])
 
       raise "\nThere is a problem with the gCloud configuration: \n #{JSON.pretty_generate(options)}" if @dataset.nil?
+
+      @itemized = @dataset.table('forge_itemized') || @dataset.create_table('forge_itemized') do |table|
+                                                        table.name        = 'Itemied dependencies between modules'
+                                                        table.description = 'A list of all types/classes/functions used by each module and where they come from'
+                                                        table.schema do |s|
+                                                          s.string  "module",  mode: :required
+                                                          s.string  "version", mode: :required
+                                                          s.string  "source"
+                                                          s.string  "kind",    mode: :required
+                                                          s.string  "element", mode: :required
+                                                          s.integer "count",   mode: :required
+                                                        end
+                                                      end
     end
 
     def truncate(entity)
@@ -72,6 +85,7 @@ class Mvp
               s.boolean   "puppet_4x"
               s.boolean   "puppet_5x"
               s.boolean   "puppet_6x"
+              s.boolean   "puppet_99x"
               s.string    "superseded_by"
               s.string    "deprecated_for"
               s.timestamp "deprecated_at"
@@ -127,6 +141,7 @@ class Mvp
               s.boolean   "puppet_4x"
               s.boolean   "puppet_5x"
               s.boolean   "puppet_6x"
+              s.boolean   "puppet_99x"
               s.string    "dependencies",     mode: :repeated
               s.string    "file_uri",         mode: :required
               s.string    "file_md5",         mode: :required
@@ -139,6 +154,7 @@ class Mvp
             end
           end
 
+          sleep 5 # this allows BigQuery time to flush schema changes
         end
       rescue => e
         $logger.error e.message
@@ -238,6 +254,13 @@ class Mvp
       end
     end
 
+    def version_itemized?(mod, version)
+      str = "SELECT version FROM forge_itemized WHERE name = '#{mod}' UNIQUE"
+      versions = @dataset.query(str).map {|row| row[:version] } rescue []
+
+      versions.include? version
+    end
+
     def test()
       require 'pry'
       binding.pry

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: puppet-community-mvp
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.3
 platform: ruby
 authors:
 - Ben Ford
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-09-06 00:00:00.000000000 Z
+date: 2018-10-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -139,6 +139,7 @@ files:
 - bin/mvp
 - lib/mvp.rb
 - lib/mvp/downloader.rb
+- lib/mvp/itemizer.rb
 - lib/mvp/monkeypatches.rb
 - lib/mvp/runner.rb
 - lib/mvp/stats.rb