rubygems-mini_mirror 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +5 -0
- data/Gemfile +4 -0
- data/LICENSE +22 -0
- data/README.md +129 -0
- data/Rakefile +6 -0
- data/TODO.md +7 -0
- data/lib/rubygems/array_ext.rb +16 -0
- data/lib/rubygems/mini_mirror.rb +25 -0
- data/lib/rubygems/mini_mirror/cli.rb +56 -0
- data/lib/rubygems/mini_mirror/command.rb +62 -0
- data/lib/rubygems/mini_mirror/dependency.rb +25 -0
- data/lib/rubygems/mini_mirror/fetcher.rb +58 -0
- data/lib/rubygems/mini_mirror/finder.rb +56 -0
- data/lib/rubygems/mini_mirror/pool.rb +21 -0
- data/lib/rubygems/mini_mirror/resource.rb +27 -0
- data/lib/rubygems/mini_mirror/resource_handler.rb +62 -0
- data/lib/rubygems/mini_mirror/resources.rb +9 -0
- data/lib/rubygems/mini_mirror/resources/base_file.rb +20 -0
- data/lib/rubygems/mini_mirror/resources/ruby_file.rb +16 -0
- data/lib/rubygems/mini_mirror/resources/yaml.rb +21 -0
- data/lib/rubygems/mini_mirror/runner.rb +109 -0
- data/lib/rubygems/mini_mirror/version.rb +5 -0
- data/lib/rubygems_plugin.rb +5 -0
- data/rubygems-mini_mirror.gemspec +41 -0
- data/rubygems-mini_mirror.tmproj +140 -0
- metadata +121 -0
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
(The MIT License)
|
2
|
+
|
3
|
+
Copyright (c) 2011 hallelujah
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,129 @@
|
|
1
|
+
# Rubygems Mini Mirror
|
2
|
+
|
3
|
+
This gem aims at providing a way to mirror only some gems you want from rubygems.
|
4
|
+
|
5
|
+
Create a minigems file
|
6
|
+
|
7
|
+
```ruby
|
8
|
+
# minigems
|
9
|
+
source :gemcutter
|
10
|
+
gem 'rails', ['~> 1.2.0','>= 3.0']
|
11
|
+
gem 'mislav-will_paginate', ['= 2.3.11'], :source => 'http://gems.github.com'
|
12
|
+
# You can also provide a resource file which provide a list of gems
|
13
|
+
# The filetype is guessed by the extension of the file and can be overriden.
|
14
|
+
# :type option can be :yaml, :ruby
|
15
|
+
resource :path => '/Users/hallelujah/repos/mini_gems.yml', :type => 'yaml'
|
16
|
+
```
|
17
|
+
|
18
|
+
The princips
|
19
|
+
------------
|
20
|
+
|
21
|
+
This nice DSL is mainly inspired by Bundler, so it should be familiar to you.
|
22
|
+
|
23
|
+
```ruby
|
24
|
+
# Fetching rails with version 1.2.3 and 2.3.5 with runtime and development dependencies
|
25
|
+
# By default development dependencies are not fetched
|
26
|
+
# *WARNING* fetching dependencies may be a slow process !! Use it with caution.
|
27
|
+
gem 'rails', ['= 1.2.3', '2.3.5'], :development => true
|
28
|
+
|
29
|
+
# If you just want to fetch development dependencies for 1.2.3 version, you need separated definitions :
|
30
|
+
gem 'rails', ['= 1.2.3', '2.3.5'], :development => true
|
31
|
+
gem 'rails', ['2.3.5']
|
32
|
+
|
33
|
+
# To add another resource file :
|
34
|
+
resource :path => 'development_gems.yml'
|
35
|
+
```
|
36
|
+
|
37
|
+
Building your own resource handler is also easy
|
38
|
+
|
39
|
+
```ruby
|
40
|
+
# You can build your own resource handler :
|
41
|
+
# my_resource_handler.rb
|
42
|
+
|
43
|
+
require 'mysql'
|
44
|
+
require 'activerecord'
|
45
|
+
class MyResourceHandler
|
46
|
+
|
47
|
+
# db_gems table structure
|
48
|
+
# name: string
|
49
|
+
# requirements: text
|
50
|
+
# sources: text
|
51
|
+
# development: boolean
|
52
|
+
class DbGem < ActiveRecord::Base
|
53
|
+
serialize :sources, Array
|
54
|
+
serialize :requirements, Array
|
55
|
+
end
|
56
|
+
|
57
|
+
include Gem::MiniMirror::Resource
|
58
|
+
register :type => 'activerecord', :ext => ['.mysql']
|
59
|
+
|
60
|
+
def initialize(runner, options)
|
61
|
+
super
|
62
|
+
@config = YAML.load_file(@path).with_indifferent_access
|
63
|
+
sources(*options[:sources])
|
64
|
+
DbGem.establish_connection @config[:db]
|
65
|
+
end
|
66
|
+
|
67
|
+
def load!
|
68
|
+
super
|
69
|
+
DbGem.all.each do |g|
|
70
|
+
gem g.name, g.requirements, :source => g.sources, :development => g.development?
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
```
|
75
|
+
|
76
|
+
```yaml
|
77
|
+
# database.yml
|
78
|
+
|
79
|
+
sources: :gemcutter
|
80
|
+
db:
|
81
|
+
adapter: mysql
|
82
|
+
database: mini_mirror
|
83
|
+
host: localhost
|
84
|
+
username: root
|
85
|
+
password:
|
86
|
+
```
|
87
|
+
|
88
|
+
```ruby
|
89
|
+
# Usage :
|
90
|
+
require 'rubygems/mini_mirror'
|
91
|
+
require 'my_resource_handler'
|
92
|
+
Gem::MiniMirror::Runner.run :path => 'database.yml', :type => 'activerecord'
|
93
|
+
```
|
94
|
+
|
95
|
+
By gem command
|
96
|
+
--------------
|
97
|
+
|
98
|
+
It provides a gem command
|
99
|
+
|
100
|
+
```bash
|
101
|
+
gem install rubygems-mini_mirror
|
102
|
+
gem mini_mirror help
|
103
|
+
gem mini_mirror -c your_gems_resource.rb -p 5 -m /home/mirrors/rubygems
|
104
|
+
# Generate index
|
105
|
+
gem generate_index --directory=/home/mirrors/rubygems --modern
|
106
|
+
```
|
107
|
+
|
108
|
+
# Thanks
|
109
|
+
|
110
|
+
Thank for the rubygems team for their work on rubygems-mirror which inspired a lot this gem.
|
111
|
+
Some parts of code of this gem are copied from [rubygems-mirror](https://github.com/rubygems/rubygems-mirror) gem.
|
112
|
+
|
113
|
+
|
114
|
+
# Note on Patches/Pull Requests
|
115
|
+
|
116
|
+
* Fork the project.
|
117
|
+
|
118
|
+
* Make your feature addition or bug fix.
|
119
|
+
|
120
|
+
* Add tests for it. This is important so I don’t break it in a future version unintentionally.
|
121
|
+
|
122
|
+
* Commit, do not mess with rakefile, version, or history. (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
|
123
|
+
|
124
|
+
* Send me a pull request. Bonus points for topic branches.
|
125
|
+
|
126
|
+
# Copyright
|
127
|
+
|
128
|
+
Copyright © 2011 Ramihajamalala Hery. See LICENSE for details
|
129
|
+
|
data/Rakefile
ADDED
data/TODO.md
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
class Array
|
2
|
+
# Courtesy from activerecord extract_options
|
3
|
+
#
|
4
|
+
# Extracts options from a set of arguments. Removes and returns the last
|
5
|
+
# element in the array if it's a hash, otherwise returns a blank hash.
|
6
|
+
#
|
7
|
+
# def options(*args)
|
8
|
+
# args.extract_options!
|
9
|
+
# end
|
10
|
+
#
|
11
|
+
# options(1, 2) # => {}
|
12
|
+
# options(1, 2, :a => :b) # => {:a=>:b}
|
13
|
+
def extract_options!
|
14
|
+
last.is_a?(::Hash) ? pop : {}
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require "rubygems/mini_mirror/version"
|
3
|
+
require "rubygems/array_ext"
|
4
|
+
require 'yaml'
|
5
|
+
|
6
|
+
module Gem
|
7
|
+
module MiniMirror
|
8
|
+
POOL_SIZE = 10
|
9
|
+
autoload :Pool, 'rubygems/mini_mirror/pool'
|
10
|
+
autoload :Fetcher, 'rubygems/mini_mirror/fetcher'
|
11
|
+
autoload :Runner, 'rubygems/mini_mirror/runner'
|
12
|
+
autoload :Cli, 'rubygems/mini_mirror/cli'
|
13
|
+
autoload :Finder, 'rubygems/mini_mirror/finder'
|
14
|
+
autoload :Resource, 'rubygems/mini_mirror/resource'
|
15
|
+
autoload :Dependency, 'rubygems/mini_mirror/dependency'
|
16
|
+
autoload :ResourceHandler, 'rubygems/mini_mirror/resource_handler'
|
17
|
+
autoload :Resources, 'rubygems/mini_mirror/resources'
|
18
|
+
autoload :Runner, 'rubygems/mini_mirror/runner'
|
19
|
+
|
20
|
+
|
21
|
+
def self.resources_handler
|
22
|
+
ResourceHandler.instance
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
module Gem
|
2
|
+
module MiniMirror
|
3
|
+
module Cli
|
4
|
+
|
5
|
+
# Load default gem configs of the system
|
6
|
+
Gem::ConfigFile.new []
|
7
|
+
|
8
|
+
DEFAULT_SOURCES = {:gemcutter => ['http://gemcutter.org'], :github => ['http://gems.github.com'], :default => Gem.sources}
|
9
|
+
|
10
|
+
def initialize(runner, options)
|
11
|
+
@options = options
|
12
|
+
@runner ||= runner
|
13
|
+
@sources = nil
|
14
|
+
@dependencies ||= []
|
15
|
+
end
|
16
|
+
|
17
|
+
def gem(name, *reqs)
|
18
|
+
options = reqs.extract_options!
|
19
|
+
reqs.flatten!
|
20
|
+
if reqs.empty?
|
21
|
+
reqs = ['>=0']
|
22
|
+
end
|
23
|
+
reqs.each do |req|
|
24
|
+
@dependencies << Gem::MiniMirror::Dependency.new(name, req, options.delete(:source) || @sources, options)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def source(*sources)
|
29
|
+
@sources ||= []
|
30
|
+
@sources |= sources.map{|s| DEFAULT_SOURCES[s] || s }.flatten
|
31
|
+
end
|
32
|
+
|
33
|
+
def resource(options={})
|
34
|
+
errors = catch(:resource_load_error) do
|
35
|
+
@runner.load_resource!(options)
|
36
|
+
nil
|
37
|
+
end
|
38
|
+
if errors
|
39
|
+
handle_error(errors)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def load!
|
44
|
+
return if @loaded
|
45
|
+
@loaded = true
|
46
|
+
end
|
47
|
+
|
48
|
+
protected
|
49
|
+
def handle_error(errors)
|
50
|
+
# MiniMirror.ui.warn(errors.inspect)
|
51
|
+
warn errors.inspect
|
52
|
+
end
|
53
|
+
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
require 'rubygems/mini_mirror'
|
2
|
+
require 'rubygems/command'
|
3
|
+
require 'yaml'
|
4
|
+
|
5
|
+
class Gem::Commands::MiniMirrorCommand < Gem::Command
|
6
|
+
SUPPORTS_INFO_SIGNAL = Signal.list['INFO']
|
7
|
+
|
8
|
+
def initialize
|
9
|
+
super 'mirror', 'Mirror a gem repository'
|
10
|
+
add_option '-c', '--config-file=FILE', 'the main config file' do |file, options|
|
11
|
+
options[:path] = file
|
12
|
+
end
|
13
|
+
add_option '-t', '--file-type=FILETYPE', 'file type of config file if not guessed' do |t, options|
|
14
|
+
options[:type] = t
|
15
|
+
end
|
16
|
+
add_option '-d', '--cd=DIR', 'directory to cd into before running the command' do |dir,options|
|
17
|
+
options[:cd] = dir
|
18
|
+
end
|
19
|
+
add_option '-r', '--require=lib1,lib2', Array, 'A list of libs to require before running the command' do |libs, options|
|
20
|
+
options[:libs] ||= []
|
21
|
+
options[:libs].push(*libs)
|
22
|
+
end
|
23
|
+
add_option '-m', '--mirror-dir=MIRROR_DIR', 'The mirror root directory : whre to download gem files. Default to <Gem.user_home>/mirror' do |dir,options|
|
24
|
+
options[:gems_dir] = dir
|
25
|
+
end
|
26
|
+
add_option '-p', '--pool-size=POOLSIZE', Integer, 'The number of threads in the pool : fro fetching concurrency. Default 10' do |pool, options|
|
27
|
+
options[:pool_size] = pool
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def description # :nodoc:
|
32
|
+
<<-EOF
|
33
|
+
The mini_mirror command mirrors remote gem
|
34
|
+
repositories to a local path. Not all the repository of rubygem is mirrored but only those you have chosen with their dependencies
|
35
|
+
|
36
|
+
See https://github.com/hallelujah/rubygems-mini_mirror
|
37
|
+
|
38
|
+
EOF
|
39
|
+
end
|
40
|
+
|
41
|
+
def execute
|
42
|
+
Dir.chdir(options.delete(:cd) || '.') do
|
43
|
+
if options[:requires]
|
44
|
+
options[:requires].each do |lib|
|
45
|
+
require lib
|
46
|
+
end
|
47
|
+
end
|
48
|
+
mirror = Gem::MiniMirror::Runner.new(options)
|
49
|
+
mirror.load_resource! options
|
50
|
+
say "Total gems: #{mirror.gems.size}"
|
51
|
+
num_to_fetch = mirror.gems_to_fetch.size
|
52
|
+
progress = ui.progress_reporter num_to_fetch, "Fetching #{num_to_fetch} gems"
|
53
|
+
trap(:INFO) { puts "Fetched: #{progress.count}/#{num_to_fetch}" } if SUPPORTS_INFO_SIGNAL
|
54
|
+
mirror.update_gems { progress.updated true }
|
55
|
+
num_to_delete = mirror.gems_to_delete.size
|
56
|
+
progress = ui.progress_reporter num_to_delete, "Deleting #{num_to_delete} gems"
|
57
|
+
trap(:INFO) { puts "Fetched: #{progress.count}/#{num_to_delete}" } if SUPPORTS_INFO_SIGNAL
|
58
|
+
mirror.delete_gems { progress.updated true }
|
59
|
+
end
|
60
|
+
|
61
|
+
end
|
62
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module Gem
|
2
|
+
module MiniMirror
|
3
|
+
class Dependency
|
4
|
+
|
5
|
+
attr_reader :dependency, :sources
|
6
|
+
|
7
|
+
def initialize(name, requirements, *args)
|
8
|
+
options = args.extract_options!
|
9
|
+
srcs = args.empty? ? nil : args.flatten
|
10
|
+
@sources = srcs || Gem.sources
|
11
|
+
@dependency = Gem::Dependency.new(name,requirements)
|
12
|
+
@development = !! options[:development]
|
13
|
+
end
|
14
|
+
|
15
|
+
def development?
|
16
|
+
@development
|
17
|
+
end
|
18
|
+
|
19
|
+
def method_missing(*args)
|
20
|
+
@dependency.send(*args)
|
21
|
+
end
|
22
|
+
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
# This code is part of rubygems rubygems-mirror
|
2
|
+
# https://github.com/rubygems/rubygems-mirror
|
3
|
+
require 'net/http'
|
4
|
+
require 'net/http/persistent'
|
5
|
+
|
6
|
+
class Gem::MiniMirror::Fetcher
|
7
|
+
# TODO beef
|
8
|
+
class Error < StandardError; end
|
9
|
+
|
10
|
+
def initialize
|
11
|
+
@http = Net::HTTP::Persistent.new(self.class.name, :ENV)
|
12
|
+
end
|
13
|
+
|
14
|
+
# Fetch a source path under the base uri, and put it in the same or given
|
15
|
+
# destination path under the base path.
|
16
|
+
def fetch(uri, path)
|
17
|
+
modified_time = File.exists?(path) && File.stat(path).mtime.rfc822
|
18
|
+
|
19
|
+
req = Net::HTTP::Get.new URI.parse(uri).path
|
20
|
+
req.add_field 'If-Modified-Since', modified_time if modified_time
|
21
|
+
|
22
|
+
@http.request URI(uri), req do |resp|
|
23
|
+
return handle_response(resp, path)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
# Handle an http response, follow redirects, etc. returns true if a file was
|
28
|
+
# downloaded, false if a 304. Raise Error on unknown responses.
|
29
|
+
def handle_response(resp, path)
|
30
|
+
case resp.code.to_i
|
31
|
+
when 304
|
32
|
+
when 302
|
33
|
+
fetch resp['location'], path
|
34
|
+
when 200
|
35
|
+
write_file(resp, path)
|
36
|
+
when 403, 404
|
37
|
+
warn "#{resp.code} on #{File.basename(path)}"
|
38
|
+
else
|
39
|
+
raise Error, "unexpected response #{resp.inspect}"
|
40
|
+
end
|
41
|
+
# TODO rescue http errors and reraise cleanly
|
42
|
+
end
|
43
|
+
|
44
|
+
# Efficiently writes an http response object to a particular path. If there
|
45
|
+
# is an error, it will remove the target file.
|
46
|
+
def write_file(resp, path)
|
47
|
+
FileUtils.mkdir_p File.dirname(path)
|
48
|
+
File.open(path, 'wb') do |output|
|
49
|
+
resp.read_body { |chunk| output << chunk }
|
50
|
+
end
|
51
|
+
true
|
52
|
+
ensure
|
53
|
+
# cleanup incomplete files, rescue perm errors etc, they're being
|
54
|
+
# raised already.
|
55
|
+
File.delete(path) rescue nil if $!
|
56
|
+
end
|
57
|
+
|
58
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
module Gem
|
2
|
+
module MiniMirror
|
3
|
+
module Finder
|
4
|
+
|
5
|
+
def initialize(options = {})
|
6
|
+
@spec_fetcher = Gem::SpecFetcher.fetcher
|
7
|
+
super()
|
8
|
+
end
|
9
|
+
|
10
|
+
def find_all_specs
|
11
|
+
@dependencies.each do |dep|
|
12
|
+
with_sources dep.sources do
|
13
|
+
found, errors = @spec_fetcher.fetch_with_errors dep, true, false
|
14
|
+
found.each do |spec,source_uri|
|
15
|
+
next if is_in_specs?(spec)
|
16
|
+
add_to_specs(spec,source_uri)
|
17
|
+
add_to_deps(*spec.runtime_dependencies)
|
18
|
+
add_to_deps(*spec.development_dependencies) if dep.development?
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def add_to_specs(spec, source_uri)
|
25
|
+
@specs.push([spec, source_uri])
|
26
|
+
@specs_list[spec.platform.to_s][spec.name.to_s][spec.version.to_s] = true
|
27
|
+
end
|
28
|
+
|
29
|
+
def is_in_specs?(spec)
|
30
|
+
@specs_list.key?(spec.platform.to_s) && @specs_list[spec.platform.to_s].key?(spec.name.to_s) && @specs_list[spec.platform.to_s][spec.name.to_s].has_key?(spec.version.to_s)
|
31
|
+
end
|
32
|
+
|
33
|
+
def add_to_deps(*deps)
|
34
|
+
deps.each do |dep|
|
35
|
+
next if is_in_deps?(dep)
|
36
|
+
dep = Gem::MiniMirror::Dependency.new(dep.name, dep.requirement,dep.respond_to?(:sources) ? dep.sources : Gem.sources, {:development => dep.respond_to?(:development?) ? dep.development? : false})
|
37
|
+
@dependencies_list[dep.name.to_s][dep.requirement.to_s] = true
|
38
|
+
@dependencies.push(dep)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def is_in_deps?(dep)
|
43
|
+
@dependencies_list[dep.name.to_s][dep.requirement.to_s] == true
|
44
|
+
end
|
45
|
+
|
46
|
+
|
47
|
+
def with_sources(srcs,&block)
|
48
|
+
before_sources = Gem.sources
|
49
|
+
Gem::sources= srcs
|
50
|
+
block.call
|
51
|
+
Gem.sources= before_sources
|
52
|
+
end
|
53
|
+
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'thread'
|
2
|
+
class Gem::MiniMirror::Pool
|
3
|
+
def initialize(size)
|
4
|
+
@size = size
|
5
|
+
@queue = Queue.new
|
6
|
+
end
|
7
|
+
|
8
|
+
def job(&blk)
|
9
|
+
@queue << blk
|
10
|
+
end
|
11
|
+
|
12
|
+
def run_til_done
|
13
|
+
threads = Array.new(@size) do
|
14
|
+
Thread.new { @queue.pop.call while true }
|
15
|
+
end
|
16
|
+
until @queue.empty? && @queue.num_waiting == @size
|
17
|
+
threads.each { |t| t.join(0.1) }
|
18
|
+
end
|
19
|
+
threads.each { |t| t.kill }
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module Gem
|
2
|
+
module MiniMirror
|
3
|
+
|
4
|
+
module Resource
|
5
|
+
|
6
|
+
def self.included(base)
|
7
|
+
base.class_eval do
|
8
|
+
attr_accessor :sources
|
9
|
+
attr_reader :dependencies
|
10
|
+
include Gem::MiniMirror::Cli
|
11
|
+
extend ClassMethods
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
module ClassMethods
|
16
|
+
def register(options={})
|
17
|
+
Gem::MiniMirror.resources_handler.add(self, options[:type], options[:ext])
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def tag
|
22
|
+
@options.to_s
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
require 'singleton'
|
2
|
+
module Gem
|
3
|
+
module MiniMirror
|
4
|
+
class ResourceHandler
|
5
|
+
class AlreadyRegistered < Exception
|
6
|
+
def initialize(type)
|
7
|
+
@type = type
|
8
|
+
end
|
9
|
+
def message
|
10
|
+
"already registered #@type"
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
class HandlerNotFound < Exception
|
15
|
+
def initialize(options = {})
|
16
|
+
@type = options[:type]
|
17
|
+
@extension = options[:ext]
|
18
|
+
end
|
19
|
+
|
20
|
+
def message
|
21
|
+
"could not find handler for file extension '#@extension'" +
|
22
|
+
@type.to_s.empty? ? "" : " or for file type #@type"
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
include Singleton
|
27
|
+
|
28
|
+
def initialize
|
29
|
+
@handlers = {}
|
30
|
+
@handlers_by_file_extension = {}
|
31
|
+
end
|
32
|
+
|
33
|
+
def add(klass, type, exts = [])
|
34
|
+
raise AlreadyRegistered, type if @handlers.key?(type.to_s)
|
35
|
+
@handlers[type.to_s] = klass
|
36
|
+
exts.each do |ext|
|
37
|
+
@handlers_by_file_extension[ext.to_s] = klass unless ext.to_s.empty?
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def find_handler_by_extension(ext)
|
42
|
+
@handlers_by_file_extension[ext.to_s]
|
43
|
+
end
|
44
|
+
|
45
|
+
def find_handler_by_type(type)
|
46
|
+
@handlers[type.to_s]
|
47
|
+
end
|
48
|
+
|
49
|
+
def find(options = {})
|
50
|
+
unless options.key?(:ext)
|
51
|
+
if path = options[:path]
|
52
|
+
options[:ext] = File.extname(path)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
handler = find_handler_by_type(options[:type]) || find_handler_by_extension(options[:ext])
|
56
|
+
raise HandlerNotFound, options unless handler
|
57
|
+
handler
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
end
|
62
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module Gem
|
2
|
+
module MiniMirror
|
3
|
+
module Resources
|
4
|
+
class BaseFile
|
5
|
+
include Gem::MiniMirror::Resource
|
6
|
+
|
7
|
+
def initialize(runner, options)
|
8
|
+
path = File.expand_path(options[:path])
|
9
|
+
throw :resource_load_error, :path => path unless File.exist?(path)
|
10
|
+
@path = path
|
11
|
+
super
|
12
|
+
end
|
13
|
+
|
14
|
+
def tag
|
15
|
+
'path-' + @path
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
module Gem
|
3
|
+
module MiniMirror
|
4
|
+
module Resources
|
5
|
+
class Yaml < BaseFile
|
6
|
+
register(:type => 'yaml', :ext => ['.yml', '.yaml'])
|
7
|
+
|
8
|
+
def load!
|
9
|
+
super
|
10
|
+
config = YAML.load_file(@path)
|
11
|
+
config.each do |instructs|
|
12
|
+
instructs.each do |k, args|
|
13
|
+
send(k, *args)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,109 @@
|
|
1
|
+
require 'rubygems/mini_mirror/resources'
|
2
|
+
module Gem
|
3
|
+
module MiniMirror
|
4
|
+
class Runner
|
5
|
+
|
6
|
+
include Gem::MiniMirror::Finder
|
7
|
+
attr_reader :specs, :resource_handler
|
8
|
+
DEFAULT_GEMS_DIR = File.join(Gem.user_home, '.gem', 'mirror')
|
9
|
+
|
10
|
+
def initialize(options = {})
|
11
|
+
@resource_handler = ResourceHandler.instance
|
12
|
+
@dependencies = []
|
13
|
+
@specs_list = Hash.new{|h,k| h[k] = Hash.new{ |h1,k1| h1[k1] = Hash.new}}
|
14
|
+
@specs = []
|
15
|
+
@dependencies_list = Hash.new{|h,k| h[k] = Hash.new}
|
16
|
+
@to_fetch = {}
|
17
|
+
@resources = []
|
18
|
+
@resources_signatures = {}
|
19
|
+
@pool = Gem::MiniMirror::Pool.new(options[:pool_size] || Gem::MiniMirror::POOL_SIZE)
|
20
|
+
@gems_dir = options[:gems_dir] || DEFAULT_GEMS_DIR
|
21
|
+
@fetcher = Gem::MiniMirror::Fetcher.new
|
22
|
+
super
|
23
|
+
end
|
24
|
+
|
25
|
+
def load_resource!(options)
|
26
|
+
klass = @resource_handler.find(options)
|
27
|
+
resource = klass.new(self, options)
|
28
|
+
unless @resources_signatures[resource.tag]
|
29
|
+
resource.load!
|
30
|
+
@resources << resource
|
31
|
+
@resources_signatures[resource.tag] = true
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def from(*args)
|
36
|
+
File.join(*args)
|
37
|
+
end
|
38
|
+
|
39
|
+
def to(*args)
|
40
|
+
File.join(@gems_dir,*args)
|
41
|
+
end
|
42
|
+
|
43
|
+
def existing_gems
|
44
|
+
Dir[to('gems','.*gem')].entries.map{|f| File.basename(f)}
|
45
|
+
end
|
46
|
+
|
47
|
+
def gems_with_sources
|
48
|
+
load_all!
|
49
|
+
@gems_with_sources ||= Hash[*specs.map{|s,src| [s.full_name + '.gem', src] }.flatten]
|
50
|
+
end
|
51
|
+
|
52
|
+
def gems
|
53
|
+
@gems ||= gems_with_sources.keys
|
54
|
+
end
|
55
|
+
|
56
|
+
def gems_to_fetch
|
57
|
+
gems - existing_gems
|
58
|
+
end
|
59
|
+
|
60
|
+
def gems_to_delete
|
61
|
+
existing_gems - gems
|
62
|
+
end
|
63
|
+
|
64
|
+
def update_gems
|
65
|
+
gems_to_fetch.each do |g|
|
66
|
+
@pool.job do
|
67
|
+
@fetcher.fetch(from(gems_with_sources[g],'gems',g),to('gems',g))
|
68
|
+
yield
|
69
|
+
end
|
70
|
+
end
|
71
|
+
@pool.run_til_done
|
72
|
+
end
|
73
|
+
|
74
|
+
def delete_gems
|
75
|
+
gems_to_delete.each do |g|
|
76
|
+
@pool.job do
|
77
|
+
File.delete(to('gems',g))
|
78
|
+
yield
|
79
|
+
end
|
80
|
+
end
|
81
|
+
@pool.run_til_done
|
82
|
+
end
|
83
|
+
|
84
|
+
def update
|
85
|
+
update_gems
|
86
|
+
delete_gems
|
87
|
+
end
|
88
|
+
|
89
|
+
def self.run(file, options = {})
|
90
|
+
runner = new(options)
|
91
|
+
runner.load_resource! :path => file, :type => 'ruby'
|
92
|
+
runner.update
|
93
|
+
end
|
94
|
+
|
95
|
+
protected
|
96
|
+
def load_all!
|
97
|
+
return if @loaded
|
98
|
+
@resources.each do |r|
|
99
|
+
add_to_deps(*r.dependencies)
|
100
|
+
end
|
101
|
+
@loaded = true
|
102
|
+
find_all_specs
|
103
|
+
end
|
104
|
+
|
105
|
+
|
106
|
+
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
$:.push File.expand_path("../lib", __FILE__)
|
3
|
+
require "rubygems/mini_mirror/version"
|
4
|
+
|
5
|
+
Gem::Specification.new do |s|
|
6
|
+
s.name = "rubygems-mini_mirror"
|
7
|
+
s.version = Gem::MiniMirror::VERSION
|
8
|
+
s.authors = ["Ramihajamalala Hery"]
|
9
|
+
s.email = ["hery@rails-royce.org"]
|
10
|
+
s.homepage = ""
|
11
|
+
s.summary = %q{A rubygems mini mirror}
|
12
|
+
s.description = %q{
|
13
|
+
Mirror some version of gems with Gem::Version DSL
|
14
|
+
Create a mini_gem file and add this to it :
|
15
|
+
source :gemcutter
|
16
|
+
gem 'rails', ['~> 1.2.0', '>= 3.0']
|
17
|
+
|
18
|
+
or
|
19
|
+
source :gemcutter
|
20
|
+
resource :path => '/your_path/mini_gem.yml'
|
21
|
+
|
22
|
+
# /your_path/mini_gem.yml
|
23
|
+
|
24
|
+
gem:
|
25
|
+
- rails:
|
26
|
+
-
|
27
|
+
- '~> 1.2.0'
|
28
|
+
- '>= 3.0'
|
29
|
+
|
30
|
+
It will solve the dependencies for you so you don't have to write an exhaustive list of the gems you want to mirror
|
31
|
+
}
|
32
|
+
|
33
|
+
s.rubyforge_project = "rubygems-mini_mirror"
|
34
|
+
s.add_runtime_dependency 'net-http-persistent', '>=1.2.5'
|
35
|
+
s.add_development_dependency 'rake'
|
36
|
+
|
37
|
+
s.files = `git ls-files`.split("\n")
|
38
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
39
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
40
|
+
s.require_paths = ["lib"]
|
41
|
+
end
|
@@ -0,0 +1,140 @@
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
2
|
+
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
3
|
+
<plist version="1.0">
|
4
|
+
<dict>
|
5
|
+
<key>currentDocument</key>
|
6
|
+
<string>lib/rubygems/mini_mirror/cli.rb</string>
|
7
|
+
<key>documents</key>
|
8
|
+
<array>
|
9
|
+
<dict>
|
10
|
+
<key>expanded</key>
|
11
|
+
<true/>
|
12
|
+
<key>name</key>
|
13
|
+
<string>rubygems-mini_mirror</string>
|
14
|
+
<key>regexFolderFilter</key>
|
15
|
+
<string>!.*/(\.[^/]*|CVS|_darcs|_MTN|\{arch\}|blib|.*~\.nib|.*\.(framework|app|pbproj|pbxproj|xcode(proj)?|bundle))$</string>
|
16
|
+
<key>sourceDirectory</key>
|
17
|
+
<string></string>
|
18
|
+
</dict>
|
19
|
+
</array>
|
20
|
+
<key>fileHierarchyDrawerWidth</key>
|
21
|
+
<integer>262</integer>
|
22
|
+
<key>metaData</key>
|
23
|
+
<dict>
|
24
|
+
<key>lib/rubygems/mini_mirror.rb</key>
|
25
|
+
<dict>
|
26
|
+
<key>caret</key>
|
27
|
+
<dict>
|
28
|
+
<key>column</key>
|
29
|
+
<integer>30</integer>
|
30
|
+
<key>line</key>
|
31
|
+
<integer>11</integer>
|
32
|
+
</dict>
|
33
|
+
<key>firstVisibleColumn</key>
|
34
|
+
<integer>0</integer>
|
35
|
+
<key>firstVisibleLine</key>
|
36
|
+
<integer>0</integer>
|
37
|
+
</dict>
|
38
|
+
<key>lib/rubygems/mini_mirror/cli.rb</key>
|
39
|
+
<dict>
|
40
|
+
<key>caret</key>
|
41
|
+
<dict>
|
42
|
+
<key>column</key>
|
43
|
+
<integer>28</integer>
|
44
|
+
<key>line</key>
|
45
|
+
<integer>25</integer>
|
46
|
+
</dict>
|
47
|
+
<key>columnSelection</key>
|
48
|
+
<false/>
|
49
|
+
<key>firstVisibleColumn</key>
|
50
|
+
<integer>0</integer>
|
51
|
+
<key>firstVisibleLine</key>
|
52
|
+
<integer>0</integer>
|
53
|
+
<key>selectFrom</key>
|
54
|
+
<dict>
|
55
|
+
<key>column</key>
|
56
|
+
<integer>24</integer>
|
57
|
+
<key>line</key>
|
58
|
+
<integer>25</integer>
|
59
|
+
</dict>
|
60
|
+
<key>selectTo</key>
|
61
|
+
<dict>
|
62
|
+
<key>column</key>
|
63
|
+
<integer>32</integer>
|
64
|
+
<key>line</key>
|
65
|
+
<integer>25</integer>
|
66
|
+
</dict>
|
67
|
+
</dict>
|
68
|
+
<key>lib/rubygems/mini_mirror/resource.rb</key>
|
69
|
+
<dict>
|
70
|
+
<key>caret</key>
|
71
|
+
<dict>
|
72
|
+
<key>column</key>
|
73
|
+
<integer>78</integer>
|
74
|
+
<key>line</key>
|
75
|
+
<integer>16</integer>
|
76
|
+
</dict>
|
77
|
+
<key>firstVisibleColumn</key>
|
78
|
+
<integer>0</integer>
|
79
|
+
<key>firstVisibleLine</key>
|
80
|
+
<integer>0</integer>
|
81
|
+
</dict>
|
82
|
+
<key>lib/rubygems/mini_mirror/resources/ruby_file.rb</key>
|
83
|
+
<dict>
|
84
|
+
<key>caret</key>
|
85
|
+
<dict>
|
86
|
+
<key>column</key>
|
87
|
+
<integer>54</integer>
|
88
|
+
<key>line</key>
|
89
|
+
<integer>15</integer>
|
90
|
+
</dict>
|
91
|
+
<key>firstVisibleColumn</key>
|
92
|
+
<integer>0</integer>
|
93
|
+
<key>firstVisibleLine</key>
|
94
|
+
<integer>0</integer>
|
95
|
+
</dict>
|
96
|
+
<key>lib/rubygems/mini_mirror/runner.rb</key>
|
97
|
+
<dict>
|
98
|
+
<key>caret</key>
|
99
|
+
<dict>
|
100
|
+
<key>column</key>
|
101
|
+
<integer>16</integer>
|
102
|
+
<key>line</key>
|
103
|
+
<integer>16</integer>
|
104
|
+
</dict>
|
105
|
+
<key>columnSelection</key>
|
106
|
+
<false/>
|
107
|
+
<key>firstVisibleColumn</key>
|
108
|
+
<integer>0</integer>
|
109
|
+
<key>firstVisibleLine</key>
|
110
|
+
<integer>0</integer>
|
111
|
+
<key>selectFrom</key>
|
112
|
+
<dict>
|
113
|
+
<key>column</key>
|
114
|
+
<integer>10</integer>
|
115
|
+
<key>line</key>
|
116
|
+
<integer>16</integer>
|
117
|
+
</dict>
|
118
|
+
<key>selectTo</key>
|
119
|
+
<dict>
|
120
|
+
<key>column</key>
|
121
|
+
<integer>23</integer>
|
122
|
+
<key>line</key>
|
123
|
+
<integer>16</integer>
|
124
|
+
</dict>
|
125
|
+
</dict>
|
126
|
+
</dict>
|
127
|
+
<key>openDocuments</key>
|
128
|
+
<array>
|
129
|
+
<string>lib/rubygems/mini_mirror/resources/ruby_file.rb</string>
|
130
|
+
<string>lib/rubygems/mini_mirror/runner.rb</string>
|
131
|
+
<string>lib/rubygems/mini_mirror/resource.rb</string>
|
132
|
+
<string>lib/rubygems/mini_mirror.rb</string>
|
133
|
+
<string>lib/rubygems/mini_mirror/cli.rb</string>
|
134
|
+
</array>
|
135
|
+
<key>showFileHierarchyDrawer</key>
|
136
|
+
<true/>
|
137
|
+
<key>windowFrame</key>
|
138
|
+
<string>{{2192, -120}, {1648, 1200}}</string>
|
139
|
+
</dict>
|
140
|
+
</plist>
|
metadata
ADDED
@@ -0,0 +1,121 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: rubygems-mini_mirror
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
hash: 23
|
5
|
+
prerelease: false
|
6
|
+
segments:
|
7
|
+
- 1
|
8
|
+
- 0
|
9
|
+
- 0
|
10
|
+
version: 1.0.0
|
11
|
+
platform: ruby
|
12
|
+
authors:
|
13
|
+
- Ramihajamalala Hery
|
14
|
+
autorequire:
|
15
|
+
bindir: bin
|
16
|
+
cert_chain: []
|
17
|
+
|
18
|
+
date: 2011-09-26 00:00:00 +02:00
|
19
|
+
default_executable:
|
20
|
+
dependencies:
|
21
|
+
- !ruby/object:Gem::Dependency
|
22
|
+
name: net-http-persistent
|
23
|
+
prerelease: false
|
24
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ">="
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
hash: 21
|
30
|
+
segments:
|
31
|
+
- 1
|
32
|
+
- 2
|
33
|
+
- 5
|
34
|
+
version: 1.2.5
|
35
|
+
type: :runtime
|
36
|
+
version_requirements: *id001
|
37
|
+
- !ruby/object:Gem::Dependency
|
38
|
+
name: rake
|
39
|
+
prerelease: false
|
40
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ">="
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
hash: 3
|
46
|
+
segments:
|
47
|
+
- 0
|
48
|
+
version: "0"
|
49
|
+
type: :development
|
50
|
+
version_requirements: *id002
|
51
|
+
description: "\n Mirror some version of gems with Gem::Version DSL\n Create a mini_gem file and add this to it :\n source :gemcutter\n gem 'rails', ['~> 1.2.0', '>= 3.0']\n\n or\n source :gemcutter\n resource :path => '/your_path/mini_gem.yml'\n\n # /your_path/mini_gem.yml\n\n gem:\n - rails:\n -\n - '~> 1.2.0'\n - '>= 3.0'\n\n It will solve the dependencies for you so you don't have to write an exhaustive list of the gems you want to mirror\n "
|
52
|
+
email:
|
53
|
+
- hery@rails-royce.org
|
54
|
+
executables: []
|
55
|
+
|
56
|
+
extensions: []
|
57
|
+
|
58
|
+
extra_rdoc_files: []
|
59
|
+
|
60
|
+
files:
|
61
|
+
- .gitignore
|
62
|
+
- Gemfile
|
63
|
+
- LICENSE
|
64
|
+
- README.md
|
65
|
+
- Rakefile
|
66
|
+
- TODO.md
|
67
|
+
- lib/rubygems/array_ext.rb
|
68
|
+
- lib/rubygems/mini_mirror.rb
|
69
|
+
- lib/rubygems/mini_mirror/cli.rb
|
70
|
+
- lib/rubygems/mini_mirror/command.rb
|
71
|
+
- lib/rubygems/mini_mirror/dependency.rb
|
72
|
+
- lib/rubygems/mini_mirror/fetcher.rb
|
73
|
+
- lib/rubygems/mini_mirror/finder.rb
|
74
|
+
- lib/rubygems/mini_mirror/pool.rb
|
75
|
+
- lib/rubygems/mini_mirror/resource.rb
|
76
|
+
- lib/rubygems/mini_mirror/resource_handler.rb
|
77
|
+
- lib/rubygems/mini_mirror/resources.rb
|
78
|
+
- lib/rubygems/mini_mirror/resources/base_file.rb
|
79
|
+
- lib/rubygems/mini_mirror/resources/ruby_file.rb
|
80
|
+
- lib/rubygems/mini_mirror/resources/yaml.rb
|
81
|
+
- lib/rubygems/mini_mirror/runner.rb
|
82
|
+
- lib/rubygems/mini_mirror/version.rb
|
83
|
+
- lib/rubygems_plugin.rb
|
84
|
+
- rubygems-mini_mirror.gemspec
|
85
|
+
- rubygems-mini_mirror.tmproj
|
86
|
+
has_rdoc: true
|
87
|
+
homepage: ""
|
88
|
+
licenses: []
|
89
|
+
|
90
|
+
post_install_message:
|
91
|
+
rdoc_options: []
|
92
|
+
|
93
|
+
require_paths:
|
94
|
+
- lib
|
95
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
96
|
+
none: false
|
97
|
+
requirements:
|
98
|
+
- - ">="
|
99
|
+
- !ruby/object:Gem::Version
|
100
|
+
hash: 3
|
101
|
+
segments:
|
102
|
+
- 0
|
103
|
+
version: "0"
|
104
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
105
|
+
none: false
|
106
|
+
requirements:
|
107
|
+
- - ">="
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
hash: 3
|
110
|
+
segments:
|
111
|
+
- 0
|
112
|
+
version: "0"
|
113
|
+
requirements: []
|
114
|
+
|
115
|
+
rubyforge_project: rubygems-mini_mirror
|
116
|
+
rubygems_version: 1.3.7
|
117
|
+
signing_key:
|
118
|
+
specification_version: 3
|
119
|
+
summary: A rubygems mini mirror
|
120
|
+
test_files: []
|
121
|
+
|