puppet-community-mvp 0.0.2 → 0.0.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/mvp +1 -0
- data/lib/mvp/downloader.rb +5 -0
- data/lib/mvp/itemizer.rb +84 -0
- data/lib/mvp/uploader.rb +23 -0
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: f77ade2721786fdca7fd96827ca510e0404a02a8
|
4
|
+
data.tar.gz: ad32f5a43392ed7f23d49d60db3fbe308ae73e2d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: be5282a77000b433c3aedd58ddfd29ea594df089007e38e74cb8b18c5c6b576ff048b218ec30cb9f60b8291faa1b62525f8d2c79694444bb3f95df2ede98efc0
|
7
|
+
data.tar.gz: 94894492ffba9d187f9178a17d51e412d80dc7b3434681bf43da79f422c6bbb0ddddba7f347b49ed048e0134a72eefe5ff3d83a615d5c4402d809f3e8010c1ac
|
data/bin/mvp
CHANGED
@@ -76,6 +76,7 @@ optparse.parse!
|
|
76
76
|
options = (YAML.load_file(options[:config]) rescue {}).merge(options)
|
77
77
|
|
78
78
|
options[:cachedir] ||= '~/.mvp/cache'
|
79
|
+
options[:github_data] ||= '~/.mvp/github_modules.csv'
|
79
80
|
options[:forgeapi] ||= 'https://forgeapi.puppet.com'
|
80
81
|
options[:gcloud] ||= {}
|
81
82
|
options[:gcloud][:dataset] ||= 'community'
|
data/lib/mvp/downloader.rb
CHANGED
@@ -3,6 +3,7 @@ require 'httparty'
|
|
3
3
|
require 'tty-spinner'
|
4
4
|
require 'semantic_puppet'
|
5
5
|
require 'mvp/monkeypatches'
|
6
|
+
require 'mvp/itemizer'
|
6
7
|
|
7
8
|
class Mvp
|
8
9
|
class Downloader
|
@@ -10,6 +11,7 @@ class Mvp
|
|
10
11
|
@useragent = 'Puppet Community Stats Monitor'
|
11
12
|
@cachedir = options[:cachedir]
|
12
13
|
@forgeapi = options[:forgeapi] ||'https://forgeapi.puppet.com'
|
14
|
+
@itemizer = Mvp::Itemizer.new(options)
|
13
15
|
end
|
14
16
|
|
15
17
|
def mirror(entity, uploader)
|
@@ -20,6 +22,8 @@ class Mvp
|
|
20
22
|
when :modules
|
21
23
|
uploader.insert(:validations, flatten_validations(retrieve_validations(data)))
|
22
24
|
data = flatten_modules(data)
|
25
|
+
|
26
|
+
@itemizer.run!(data, uploader)
|
23
27
|
when :releases
|
24
28
|
data = flatten_releases(data)
|
25
29
|
end
|
@@ -228,6 +232,7 @@ class Mvp
|
|
228
232
|
data['puppet_4x'] = range.include? SemanticPuppet::Version.parse('4.99.99')
|
229
233
|
data['puppet_5x'] = range.include? SemanticPuppet::Version.parse('5.99.99')
|
230
234
|
data['puppet_6x'] = range.include? SemanticPuppet::Version.parse('6.99.99')
|
235
|
+
data['puppet_99x'] = range.include? SemanticPuppet::Version.parse('99.99.99') # identify unbounded ranges
|
231
236
|
end
|
232
237
|
end
|
233
238
|
|
data/lib/mvp/itemizer.rb
ADDED
@@ -0,0 +1,84 @@
|
|
1
|
+
require 'httparty'
|
2
|
+
require 'fileutils'
|
3
|
+
require 'puppet_x/binford2k/itemize'
|
4
|
+
require 'puppet_x/binford2k/itemize/runner'
|
5
|
+
|
6
|
+
class Mvp
|
7
|
+
class Itemizer
|
8
|
+
def initialize(options = {})
|
9
|
+
@useragent = 'Puppet Community Stats Monitor'
|
10
|
+
@forge = options[:forge] ||'https://forge.puppet.com'
|
11
|
+
end
|
12
|
+
|
13
|
+
def run!(data, uploader)
|
14
|
+
data.each do |mod|
|
15
|
+
modname = mod['slug']
|
16
|
+
version = mod['version']
|
17
|
+
return if uploader.version_itemized?(modname, version)
|
18
|
+
|
19
|
+
$logger.debug "Itemizing #{modname}-#{version}"
|
20
|
+
begin
|
21
|
+
rows = table(itemize(modname, version), mod)
|
22
|
+
uploader.insert(:itemized, rows) unless rows.empty?
|
23
|
+
rescue => e
|
24
|
+
$logger.error e.message
|
25
|
+
$logger.debug e.backtrace.join("\n")
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
def download(path, modname, version)
|
31
|
+
filename = "#{modname}-#{version}.tar.gz"
|
32
|
+
Dir.chdir(path) do
|
33
|
+
File.open(filename, "w") do |file|
|
34
|
+
file << HTTParty.get( "#{@forge}/v3/files/#{filename}" )
|
35
|
+
end
|
36
|
+
system("tar -xf #{filename}")
|
37
|
+
FileUtils.rm(filename)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def itemize(modname, version)
|
42
|
+
Dir.mktmpdir('mvp') do |path|
|
43
|
+
download(path, modname, version)
|
44
|
+
|
45
|
+
# not all modules have manifests
|
46
|
+
manifests = "#{path}/#{modname}-#{version}/manifests"
|
47
|
+
next {} unless File.directory?(manifests)
|
48
|
+
|
49
|
+
options = {
|
50
|
+
:manifests => [manifests],
|
51
|
+
:external => true,
|
52
|
+
}
|
53
|
+
runner = Puppet_X::Binford2k::Itemize::Runner.new(options).run!
|
54
|
+
runner.results
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
# Build a table with this schema
|
59
|
+
# module | version | source | kind | element | count
|
60
|
+
def table(itemized, data)
|
61
|
+
modname = data['slug']
|
62
|
+
version = data['version']
|
63
|
+
dependencies = data['dependencies']
|
64
|
+
|
65
|
+
itemized.map do |kind, elements|
|
66
|
+
# the kind of element comes pluralized from puppet-itemize
|
67
|
+
kind = kind.to_s
|
68
|
+
kind = kind.end_with?('ses') ? kind.chomp('es') : kind.chomp('s')
|
69
|
+
elements.map do |name, count|
|
70
|
+
# TODO: this may suffer from collisions, (module foo, function foo, for example)
|
71
|
+
depname = name.split('::').first
|
72
|
+
source = dependencies.find {|row| row.split('-').last == depname} rescue nil
|
73
|
+
|
74
|
+
{ :module => modname, :version => version, :source => source, :kind => kind, :element => name, :count => count }
|
75
|
+
end
|
76
|
+
end.flatten(1)
|
77
|
+
end
|
78
|
+
|
79
|
+
def test()
|
80
|
+
require 'pry'
|
81
|
+
binding.pry
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
data/lib/mvp/uploader.rb
CHANGED
@@ -14,6 +14,19 @@ class Mvp
|
|
14
14
|
@dataset = @bigquery.dataset(options[:gcloud][:dataset])
|
15
15
|
|
16
16
|
raise "\nThere is a problem with the gCloud configuration: \n #{JSON.pretty_generate(options)}" if @dataset.nil?
|
17
|
+
|
18
|
+
@itemized = @dataset.table('forge_itemized') || @dataset.create_table('forge_itemized') do |table|
|
19
|
+
table.name = 'Itemied dependencies between modules'
|
20
|
+
table.description = 'A list of all types/classes/functions used by each module and where they come from'
|
21
|
+
table.schema do |s|
|
22
|
+
s.string "module", mode: :required
|
23
|
+
s.string "version", mode: :required
|
24
|
+
s.string "source"
|
25
|
+
s.string "kind", mode: :required
|
26
|
+
s.string "element", mode: :required
|
27
|
+
s.integer "count", mode: :required
|
28
|
+
end
|
29
|
+
end
|
17
30
|
end
|
18
31
|
|
19
32
|
def truncate(entity)
|
@@ -72,6 +85,7 @@ class Mvp
|
|
72
85
|
s.boolean "puppet_4x"
|
73
86
|
s.boolean "puppet_5x"
|
74
87
|
s.boolean "puppet_6x"
|
88
|
+
s.boolean "puppet_99x"
|
75
89
|
s.string "superseded_by"
|
76
90
|
s.string "deprecated_for"
|
77
91
|
s.timestamp "deprecated_at"
|
@@ -127,6 +141,7 @@ class Mvp
|
|
127
141
|
s.boolean "puppet_4x"
|
128
142
|
s.boolean "puppet_5x"
|
129
143
|
s.boolean "puppet_6x"
|
144
|
+
s.boolean "puppet_99x"
|
130
145
|
s.string "dependencies", mode: :repeated
|
131
146
|
s.string "file_uri", mode: :required
|
132
147
|
s.string "file_md5", mode: :required
|
@@ -139,6 +154,7 @@ class Mvp
|
|
139
154
|
end
|
140
155
|
end
|
141
156
|
|
157
|
+
sleep 5 # this allows BigQuery time to flush schema changes
|
142
158
|
end
|
143
159
|
rescue => e
|
144
160
|
$logger.error e.message
|
@@ -238,6 +254,13 @@ class Mvp
|
|
238
254
|
end
|
239
255
|
end
|
240
256
|
|
257
|
+
def version_itemized?(mod, version)
|
258
|
+
str = "SELECT version FROM forge_itemized WHERE name = '#{mod}' UNIQUE"
|
259
|
+
versions = @dataset.query(str).map {|row| row[:version] } rescue []
|
260
|
+
|
261
|
+
versions.include? version
|
262
|
+
end
|
263
|
+
|
241
264
|
def test()
|
242
265
|
require 'pry'
|
243
266
|
binding.pry
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: puppet-community-mvp
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ben Ford
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2018-
|
11
|
+
date: 2018-10-26 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: json
|
@@ -139,6 +139,7 @@ files:
|
|
139
139
|
- bin/mvp
|
140
140
|
- lib/mvp.rb
|
141
141
|
- lib/mvp/downloader.rb
|
142
|
+
- lib/mvp/itemizer.rb
|
142
143
|
- lib/mvp/monkeypatches.rb
|
143
144
|
- lib/mvp/runner.rb
|
144
145
|
- lib/mvp/stats.rb
|