gemirro 1.5.0 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ruby.yml +29 -0
  3. data/.rubocop.yml +6 -3
  4. data/Gemfile.lock +126 -0
  5. data/MANIFEST +6 -10
  6. data/README.md +1 -1
  7. data/bin/gemirro +7 -1
  8. data/gemirro.gemspec +7 -2
  9. data/lib/gemirro/cli/index.rb +12 -3
  10. data/lib/gemirro/cli/init.rb +6 -1
  11. data/lib/gemirro/cli/update.rb +6 -0
  12. data/lib/gemirro/configuration.rb +1 -1
  13. data/lib/gemirro/gems_fetcher.rb +5 -10
  14. data/lib/gemirro/indexer.rb +396 -89
  15. data/lib/gemirro/mirror_file.rb +1 -0
  16. data/lib/gemirro/server.rb +73 -166
  17. data/lib/gemirro/utils.rb +123 -68
  18. data/lib/gemirro/version.rb +1 -1
  19. data/lib/gemirro/versions_fetcher.rb +6 -2
  20. data/lib/gemirro.rb +0 -1
  21. data/spec/gemirro/server_spec.rb +76 -47
  22. data/template/config.rb +6 -0
  23. data/template/public/dist/css/gemirro.css +25 -1
  24. data/template/public/latest_specs.4.8 +0 -0
  25. data/template/public/prerelease_specs.4.8 +0 -0
  26. data/template/public/specs.4.8 +0 -0
  27. data/views/gem.erb +46 -37
  28. data/views/index.erb +41 -33
  29. data/views/layout.erb +4 -17
  30. data/views/not_found.erb +4 -4
  31. metadata +82 -16
  32. data/lib/gemirro/cache.rb +0 -115
  33. data/spec/gemirro/cache_spec.rb +0 -32
  34. data/template/public/dist/css/bootstrap.min.css +0 -7
  35. data/template/public/dist/fonts/glyphicons-halflings-regular.eot +0 -0
  36. data/template/public/dist/fonts/glyphicons-halflings-regular.svg +0 -288
  37. data/template/public/dist/fonts/glyphicons-halflings-regular.ttf +0 -0
  38. data/template/public/dist/fonts/glyphicons-halflings-regular.woff +0 -0
  39. data/template/public/dist/fonts/glyphicons-halflings-regular.woff2 +0 -0
  40. data/template/public/dist/js/bootstrap.min.js +0 -7
@@ -10,12 +10,6 @@ module Gemirro
10
10
  # Launch Sinatra server to easily download gems.
11
11
  #
12
12
  class Server < Sinatra::Base
13
- # rubocop:disable Layout/LineLength
14
- URI_REGEXP = /^(.*)-(\d+(?:\.\d+){1,4}.*?)(?:-(x86-(?:(?:mswin|mingw)(?:32|64)).*?|java))?\.(gem(?:spec\.rz)?)$/.freeze
15
- # rubocop:enable Layout/LineLength
16
- GEMSPEC_TYPE = 'gemspec.rz'
17
- GEM_TYPE = 'gem'
18
-
19
13
  access_logger = Logger.new(Utils.configuration.server.access_log).tap do |logger|
20
14
  ::Logger.class_eval { alias_method :write, :<< }
21
15
  logger.level = ::Logger::INFO
@@ -62,7 +56,7 @@ module Gemirro
62
56
  end
63
57
 
64
58
  ##
65
- # Display information about one gem
59
+ # Display information about one gem, human readable
66
60
  #
67
61
  # @return [nil]
68
62
  #
@@ -76,7 +70,7 @@ module Gemirro
76
70
 
77
71
  ##
78
72
  # Display home page containing the list of gems already
79
- # downloaded on the server
73
+ # downloaded on the server, human readable
80
74
  #
81
75
  # @return [nil]
82
76
  #
@@ -85,13 +79,17 @@ module Gemirro
85
79
  end
86
80
 
87
81
  ##
88
- # Return gem dependencies as binary
82
+ # Return gem dependencies as marshaled binary
89
83
  #
90
84
  # @return [nil]
91
85
  #
92
86
  get '/api/v1/dependencies' do
93
87
  content_type 'application/octet-stream'
94
- query_gems.any? ? Marshal.dump(query_gems_list) : 200
88
+ if params[:gems].to_s != '' && params[:gems].to_s.split(',').any?
89
+ Marshal.dump(dependencies_loader(params[:gems].to_s.split(',')))
90
+ else
91
+ 200
92
+ end
95
93
  end
96
94
 
97
95
  ##
@@ -101,196 +99,105 @@ module Gemirro
101
99
  #
102
100
  get '/api/v1/dependencies.json' do
103
101
  content_type 'application/json'
104
- query_gems.any? ? JSON.dump(query_gems_list) : {}
102
+
103
+ return '[]' unless params[:gems]
104
+
105
+ gem_names = params[:gems].to_s
106
+ .split(',')
107
+ .map(&:strip)
108
+ .reject(&:empty?)
109
+ return '[]' if gem_names.empty?
110
+
111
+ JSON.dump(dependencies_loader(gem_names))
105
112
  end
106
113
 
107
114
  ##
108
- # Try to get all request and download files
109
- # if files aren't found.
115
+ # compact_index, Return list of available gem names
110
116
  #
111
117
  # @return [nil]
112
118
  #
113
- get('*') do |path|
114
- resource = "#{settings.public_folder}#{path}"
119
+ get '/names' do
120
+ content_type 'text/plain'
115
121
 
116
- # Try to download gem
117
- fetch_gem(resource) unless File.exist?(resource)
118
- # If not found again, return a 404
119
- return not_found unless File.exist?(resource)
122
+ content_path = Dir.glob(File.join(Gemirro.configuration.destination, 'names.*.*.list')).last
123
+ _, etag, repr_digest, _ = content_path.split('.', -4)
120
124
 
121
- send_file(resource)
125
+ headers 'etag' => etag
126
+ headers 'repr-digest' => %(sha-256="#{repr_digest}")
127
+ send_file content_path
122
128
  end
123
129
 
124
130
  ##
125
- # Try to fetch gem and download its if it's possible, and
126
- # build and install indicies.
131
+ # compact_index, Return list of gem, including versions
127
132
  #
128
- # @param [String] resource
129
- # @return [Indexer]
133
+ # @return [nil]
130
134
  #
131
- def fetch_gem(resource)
132
- return unless Utils.configuration.fetch_gem
133
-
134
- name = File.basename(resource)
135
- result = name.match(URI_REGEXP)
136
- return unless result
137
-
138
- gem_name, gem_version, gem_platform, gem_type = result.captures
139
- return unless gem_name && gem_version
135
+ get '/versions' do
136
+ content_type 'text/plain'
140
137
 
141
- begin
142
- gem = Utils.stored_gem(gem_name, gem_version, gem_platform)
143
- gem.gemspec = true if gem_type == GEMSPEC_TYPE
138
+ content_path = Dir.glob(File.join(Utils.configuration.destination, 'versions.*.*.list')).last
139
+ _, etag, repr_digest, _ = content_path.split('.', -4)
144
140
 
145
- return if Utils.gems_fetcher.gem_exists?(gem.filename(gem_version)) && gem_type == GEM_TYPE
146
- return if Utils.gems_fetcher.gemspec_exists?(gem.gemspec_filename(gem_version)) && gem_type == GEMSPEC_TYPE
147
-
148
- Utils.logger
149
- .info("Try to download #{gem_name} with version #{gem_version}")
150
- Utils.gems_fetcher.source.gems.clear
151
- Utils.gems_fetcher.source.gems.push(gem)
152
- Utils.gems_fetcher.fetch
153
-
154
- update_indexes if Utils.configuration.update_on_fetch
155
- rescue StandardError => e
156
- Utils.logger.error(e)
157
- end
141
+ headers 'etag' => etag
142
+ headers 'repr-digest' => %(sha-256="#{repr_digest}")
143
+ send_file content_path
158
144
  end
159
145
 
160
- ##
161
- # Update indexes files
146
+ # compact_index, Return gem dependencies for all versions of a gem
162
147
  #
163
- # @return [Indexer]
148
+ # @return [nil]
164
149
  #
165
- def update_indexes
166
- indexer = Gemirro::Indexer.new(Utils.configuration.destination)
167
- indexer.only_origin = true
168
- indexer.ui = ::Gem::SilentUI.new
150
+ get('/info/:gemname') do
151
+ gems = Utils.gems_collection
152
+ gem = gems.find_by_name(params[:gemname])
153
+ return not_found if gem.nil?
169
154
 
170
- Utils.logger.info('Generating indexes')
171
- indexer.update_index
172
- indexer.updated_gems.each do |gem|
173
- Utils.cache.flush_key(File.basename(gem))
174
- end
175
- rescue SystemExit => e
176
- Utils.logger.info(e.message)
177
- end
155
+ content_type 'text/plain'
178
156
 
179
- ##
180
- # Return all gems pass to query
181
- #
182
- # @return [Array]
183
- #
184
- def query_gems
185
- params[:gems].to_s.split(',')
157
+ content_path = Dir.glob(File.join(Utils.configuration.destination, 'info', "#{params[:gemname]}.*.*.list")).last
158
+ _, etag, repr_digest, _ = content_path.split('.', -4)
159
+
160
+ headers 'etag' => etag
161
+ headers 'repr-digest' => %(sha-256="#{repr_digest}")
162
+ send_file content_path
186
163
  end
187
164
 
188
165
  ##
189
- # Return gems list from query params
166
+ # Try to get all request and download files
167
+ # if files aren't found.
190
168
  #
191
- # @return [Array]
169
+ # @return [nil]
192
170
  #
193
- def query_gems_list
194
- Utils.gems_collection(false) # load collection
195
- gems = Parallel.map(query_gems, in_threads: 4) do |query_gem|
196
- gem_dependencies(query_gem)
197
- end
171
+ get('*') do |path|
172
+ resource = "#{settings.public_folder}#{path}"
198
173
 
199
- gems.flatten!
200
- gems.reject!(&:empty?)
201
- gems
174
+ # Try to download gem
175
+ Gemirro::Utils.fetch_gem(resource) unless File.exist?(resource)
176
+ # If not found again, return a 404
177
+ return not_found unless File.exist?(resource)
178
+
179
+ send_file(resource)
202
180
  end
203
181
 
204
182
  ##
205
- # List of versions and dependencies of each version
206
- # from a gem name.
183
+ # Compile fragments for /api/v1/dependencies
207
184
  #
208
- # @return [Array]
185
+ # @return [nil]
209
186
  #
210
- def gem_dependencies(gem_name)
211
- Utils.cache.cache(gem_name) do
212
- gems = Utils.gems_collection(false)
213
- gem_collection = gems.find_by_name(gem_name)
214
-
215
- return '' if gem_collection.nil?
216
-
217
- gem_collection = Parallel.map(gem_collection, in_threads: 4) do |gem|
218
- [gem, spec_for(gem.name, gem.number, gem.platform)]
219
- end
220
- gem_collection.compact!
221
-
222
- Parallel.map(gem_collection, in_threads: 4) do |gem, spec|
223
- next if spec.nil?
224
-
225
- dependencies = spec.dependencies.select do |d|
226
- d.type == :runtime
227
- end
228
-
229
- dependencies = Parallel.map(dependencies, in_threads: 4) do |d|
230
- [d.name.is_a?(Array) ? d.name.first : d.name, d.requirement.to_s]
231
- end
232
-
233
- {
234
- name: gem.name,
235
- number: gem.number,
236
- platform: gem.platform,
237
- dependencies: dependencies
238
- }
239
- end
240
- end
241
- end
242
-
243
- helpers do
244
- ##
245
- # Return gem specification from gemname and version
246
- #
247
- # @param [String] gemname
248
- # @param [String] version
249
- # @return [::Gem::Specification]
250
- #
251
- def spec_for(gemname, version, platform = 'ruby')
252
- gem = Utils.stored_gem(gemname, version.to_s, platform)
253
- gemspec_path = File.join('quick',
254
- Gemirro::Configuration.marshal_identifier,
255
- gem.gemspec_filename)
256
- spec_file = File.join(settings.public_folder,
257
- gemspec_path)
258
- fetch_gem(gemspec_path) unless File.exist?(spec_file)
259
-
260
- return unless File.exist?(spec_file)
261
-
262
- File.open(spec_file, 'r') do |uz_file|
263
- uz_file.binmode
264
- inflater = Zlib::Inflate.new
265
- begin
266
- inflate_data = inflater.inflate(uz_file.read)
267
- ensure
268
- inflater.finish
269
- inflater.close
270
- end
271
- Marshal.load(inflate_data)
187
+ def dependencies_loader(names)
188
+ names.collect do |name|
189
+ f = File.join(settings.public_folder, 'api', 'v1', 'dependencies', "#{name}.*.*.list")
190
+ Marshal.load(File.read(Dir.glob(f).last))
191
+ rescue StandardError => e
192
+ env['rack.errors'].write "Cound not open #{f}\n"
193
+ env['rack.errors'].write "#{e.message}\n"
194
+ e.backtrace.each do |err|
195
+ env['rack.errors'].write "#{err}\n"
272
196
  end
197
+ nil
273
198
  end
274
-
275
- ##
276
- # Escape string
277
- #
278
- # @param [String] string
279
- # @return [String]
280
- #
281
- def escape(string)
282
- Rack::Utils.escape_html(string)
283
- end
284
-
285
- ##
286
- # Homepage link
287
- #
288
- # @param [Gem] spec
289
- # @return [String]
290
- #
291
- def homepage(spec)
292
- URI.parse(Addressable::URI.escape(spec.homepage))
293
- end
199
+ .flatten
200
+ .compact
294
201
  end
295
202
  end
296
203
  end
data/lib/gemirro/utils.rb CHANGED
@@ -13,84 +13,52 @@ module Gemirro
13
13
  # @return [Gemirro::GemsFetcher]
14
14
  #
15
15
  class Utils
16
- attr_reader(:cache,
17
- :versions_fetcher,
18
- :gems_fetcher,
19
- :gems_collection,
20
- :stored_gems)
16
+ attr_reader(
17
+ :versions_fetcher,
18
+ :gems_fetcher,
19
+ :gems_collection,
20
+ :stored_gems
21
+ )
21
22
 
22
- ##
23
- # Cache class to store marshal and data into files
24
- #
25
- # @return [Gemirro::Cache]
26
- #
27
- def self.cache
28
- @cache ||= Gemirro::Cache
29
- .new(File.join(configuration.destination, '.cache'))
30
- end
23
+ URI_REGEXP = /^(.*)-(\d+(?:\.\d+){1,4}.*?)(?:-(x86-(?:(?:mswin|mingw)(?:32|64)).*?|java))?\.(gem(?:spec\.rz)?)$/
24
+ GEMSPEC_TYPE = 'gemspec.rz'
25
+ GEM_TYPE = 'gem'
31
26
 
32
27
  ##
33
- # Generate Gems collection from Marshal dump
28
+ # Generate Gems collection from Marshal dump - always the .local file
34
29
  #
35
- # @param [TrueClass|FalseClass] orig Fetch orig files
36
30
  # @return [Gemirro::GemVersionCollection]
37
31
  #
38
- def self.gems_collection(orig = true)
39
- @gems_collection = {} if @gems_collection.nil?
40
-
41
- is_orig = orig ? 1 : 0
42
- data = @gems_collection[is_orig]
43
- data = { files: {}, values: nil } if data.nil?
44
-
45
- file_paths = specs_files_paths(orig)
46
- has_file_changed = false
47
- Parallel.map(file_paths, in_threads: 4) do |file_path|
48
- next if data[:files].key?(file_path) &&
49
- data[:files][file_path] == File.mtime(file_path)
50
-
51
- has_file_changed = true
52
- end
32
+ def self.gems_collection
33
+ @gems_collection ||= { files: {}, values: nil }
34
+
35
+ file_paths =
36
+ %i[specs prerelease_specs].collect do |specs_file_type|
37
+ File.join(
38
+ configuration.destination,
39
+ "#{specs_file_type}.#{Gemirro::Configuration.marshal_version}.gz.local"
40
+ )
41
+ end
42
+
43
+ has_file_changed =
44
+ @gems_collection[:files] != file_paths.each_with_object({}) do |f, r|
45
+ r[f] = File.mtime(f) if File.exist?(f)
46
+ end
53
47
 
54
48
  # Return result if no file changed
55
- return data[:values] if !has_file_changed && !data[:values].nil?
49
+ return @gems_collection[:values] if !has_file_changed && !@gems_collection[:values].nil?
56
50
 
57
51
  gems = []
58
- Parallel.map(file_paths, in_threads: 4) do |file_path|
52
+
53
+ # parallel is not for mtime, it's for the Marshal.
54
+ Parallel.map(file_paths, in_threads: Utils.configuration.update_thread_count) do |file_path|
59
55
  next unless File.exist?(file_path)
60
56
 
61
57
  gems.concat(Marshal.load(Zlib::GzipReader.open(file_path).read))
62
- data[:files][file_path] = File.mtime(file_path)
58
+ @gems_collection[:files][file_path] = File.mtime(file_path)
63
59
  end
64
60
 
65
- collection = GemVersionCollection.new(gems)
66
- data[:values] = collection
67
-
68
- collection
69
- end
70
-
71
- ##
72
- # Return specs fils paths
73
- #
74
- # @param [TrueClass|FalseClass] orig Fetch orig files
75
- # @return [Array]
76
- #
77
- def self.specs_files_paths(orig = true)
78
- marshal_version = Gemirro::Configuration.marshal_version
79
- Parallel.map(specs_file_types, in_threads: 4) do |specs_file_type|
80
- File.join(configuration.destination,
81
- [specs_file_type,
82
- marshal_version,
83
- "gz#{orig ? '.orig' : ''}"].join('.'))
84
- end
85
- end
86
-
87
- ##
88
- # Return specs fils types
89
- #
90
- # @return [Array]
91
- #
92
- def self.specs_file_types
93
- %i[specs prerelease_specs]
61
+ @gems_collection[:values] = GemVersionCollection.new(gems)
94
62
  end
95
63
 
96
64
  ##
@@ -112,8 +80,7 @@ module Gemirro
112
80
  # @see Gemirro::VersionsFetcher.fetch
113
81
  #
114
82
  def self.versions_fetcher
115
- @versions_fetcher ||= Gemirro::VersionsFetcher
116
- .new(configuration.source).fetch
83
+ @versions_fetcher ||= Gemirro::VersionsFetcher.new(configuration.source).fetch
117
84
  end
118
85
 
119
86
  ##
@@ -133,13 +100,101 @@ module Gemirro
133
100
  def self.stored_gem(gem_name, gem_version, platform = 'ruby')
134
101
  platform = 'ruby' if platform.nil?
135
102
  @stored_gems ||= {}
136
- # rubocop:disable Metrics/LineLength
137
103
  @stored_gems[gem_name] = {} unless @stored_gems.key?(gem_name)
138
104
  @stored_gems[gem_name][gem_version] = {} unless @stored_gems[gem_name].key?(gem_version)
139
- @stored_gems[gem_name][gem_version][platform] ||= Gem.new(gem_name, gem_version, platform) unless @stored_gems[gem_name][gem_version].key?(platform)
140
- # rubocop:enable Metrics/LineLength
105
+ unless @stored_gems[gem_name][gem_version].key?(platform)
106
+ @stored_gems[gem_name][gem_version][platform] ||= Gem.new(gem_name, gem_version, platform)
107
+ end
141
108
 
142
109
  @stored_gems[gem_name][gem_version][platform]
143
110
  end
111
+
112
+ ##
113
+ # Return gem specification from gemname and version
114
+ #
115
+ # @param [String] gemname
116
+ # @param [String] version
117
+ # @return [::Gem::Specification]
118
+ #
119
+ def self.spec_for(gemname, version, platform)
120
+ gem = Utils.stored_gem(gemname, version.to_s, platform)
121
+
122
+ spec_file =
123
+ File.join(
124
+ configuration.destination,
125
+ 'quick',
126
+ Gemirro::Configuration.marshal_identifier,
127
+ gem.gemspec_filename
128
+ )
129
+
130
+ fetch_gem(spec_file) unless File.exist?(spec_file)
131
+
132
+ # this is a separate action
133
+ return unless File.exist?(spec_file)
134
+
135
+ File.open(spec_file, 'r') do |uz_file|
136
+ uz_file.binmode
137
+ inflater = Zlib::Inflate.new
138
+ begin
139
+ inflate_data = inflater.inflate(uz_file.read)
140
+ ensure
141
+ inflater.finish
142
+ inflater.close
143
+ end
144
+
145
+ Marshal.load(inflate_data)
146
+ end
147
+ end
148
+
149
+ ##
150
+ # Try to fetch gem and download its if it's possible, and
151
+ # build and install indicies.
152
+ #
153
+ # @param [String] resource
154
+ # @return [Indexer]
155
+ #
156
+ def self.fetch_gem(resource)
157
+ return unless Utils.configuration.fetch_gem
158
+
159
+ name = File.basename(resource)
160
+ result = name.match(URI_REGEXP)
161
+ return unless result
162
+
163
+ gem_name, gem_version, gem_platform, gem_type = result.captures
164
+ return unless gem_name && gem_version
165
+
166
+ begin
167
+ gem = Utils.stored_gem(gem_name, gem_version, gem_platform)
168
+ gem.gemspec = true if gem_type == GEMSPEC_TYPE
169
+
170
+ return if Utils.gems_fetcher.gem_exists?(gem.filename(gem_version)) && gem_type == GEM_TYPE
171
+ return if Utils.gems_fetcher.gemspec_exists?(gem.gemspec_filename(gem_version)) && gem_type == GEMSPEC_TYPE
172
+
173
+ Utils.logger.info("Try to download #{gem_name} with version #{gem_version}")
174
+ Utils.gems_fetcher.source.gems.clear
175
+ Utils.gems_fetcher.source.gems.push(gem)
176
+ Utils.gems_fetcher.fetch
177
+
178
+ update_indexes if Utils.configuration.update_on_fetch
179
+ rescue StandardError => e
180
+ Utils.logger.error(e)
181
+ end
182
+ end
183
+
184
+ ##
185
+ # Update indexes files
186
+ #
187
+ # @return [Indexer]
188
+ #
189
+ def self.update_indexes
190
+ indexer = Gemirro::Indexer.new(Utils.configuration.destination)
191
+ indexer.only_origin = true
192
+ indexer.ui = ::Gem::SilentUI.new
193
+
194
+ Utils.logger.info('Generating indexes')
195
+ indexer.update_index
196
+ rescue SystemExit => e
197
+ Utils.logger.info(e.message)
198
+ end
144
199
  end
145
200
  end
@@ -2,5 +2,5 @@
2
2
 
3
3
  # Gemirro Version
4
4
  module Gemirro
5
- VERSION = '1.5.0'
5
+ VERSION = '1.6.0'
6
6
  end
@@ -22,8 +22,10 @@ module Gemirro
22
22
  # @return [Gemirro::VersionsFile]
23
23
  #
24
24
  def fetch
25
- VersionsFile.load(read_file(Configuration.versions_file),
26
- read_file(Configuration.prerelease_versions_file, true))
25
+ VersionsFile.load(
26
+ read_file(Configuration.versions_file),
27
+ read_file(Configuration.prerelease_versions_file, true)
28
+ )
27
29
  end
28
30
 
29
31
  ##
@@ -36,6 +38,8 @@ module Gemirro
36
38
  destination = Gemirro.configuration.destination
37
39
  file_dst = File.join(destination, file)
38
40
  unless File.exist?(file_dst)
41
+ throw 'No source defined' unless @source
42
+
39
43
  File.write(file_dst, @source.fetch_versions) unless prerelease
40
44
  File.write(file_dst, @source.fetch_prerelease_versions) if prerelease
41
45
  end
data/lib/gemirro.rb CHANGED
@@ -20,7 +20,6 @@ $LOAD_PATH.unshift(File.expand_path('../', __FILE__)) unless $LOAD_PATH.include?
20
20
 
21
21
  require 'gemirro/version'
22
22
  require 'gemirro/configuration'
23
- require 'gemirro/cache'
24
23
  require 'gemirro/utils'
25
24
  require 'gemirro/gem'
26
25
  require 'gemirro/gem_version'