rubygems-mini_mirror 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- data/.gitignore +5 -0
- data/Gemfile +4 -0
- data/LICENSE +22 -0
- data/README.md +129 -0
- data/Rakefile +6 -0
- data/TODO.md +7 -0
- data/lib/rubygems/array_ext.rb +16 -0
- data/lib/rubygems/mini_mirror.rb +25 -0
- data/lib/rubygems/mini_mirror/cli.rb +56 -0
- data/lib/rubygems/mini_mirror/command.rb +62 -0
- data/lib/rubygems/mini_mirror/dependency.rb +25 -0
- data/lib/rubygems/mini_mirror/fetcher.rb +58 -0
- data/lib/rubygems/mini_mirror/finder.rb +56 -0
- data/lib/rubygems/mini_mirror/pool.rb +21 -0
- data/lib/rubygems/mini_mirror/resource.rb +27 -0
- data/lib/rubygems/mini_mirror/resource_handler.rb +62 -0
- data/lib/rubygems/mini_mirror/resources.rb +9 -0
- data/lib/rubygems/mini_mirror/resources/base_file.rb +20 -0
- data/lib/rubygems/mini_mirror/resources/ruby_file.rb +16 -0
- data/lib/rubygems/mini_mirror/resources/yaml.rb +21 -0
- data/lib/rubygems/mini_mirror/runner.rb +109 -0
- data/lib/rubygems/mini_mirror/version.rb +5 -0
- data/lib/rubygems_plugin.rb +5 -0
- data/rubygems-mini_mirror.gemspec +41 -0
- data/rubygems-mini_mirror.tmproj +140 -0
- metadata +121 -0
data/Gemfile
ADDED
data/LICENSE
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
(The MIT License)
|
2
|
+
|
3
|
+
Copyright (c) 2011 hallelujah
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining
|
6
|
+
a copy of this software and associated documentation files (the
|
7
|
+
"Software"), to deal in the Software without restriction, including
|
8
|
+
without limitation the rights to use, copy, modify, merge, publish,
|
9
|
+
distribute, sublicense, and/or sell copies of the Software, and to
|
10
|
+
permit persons to whom the Software is furnished to do so, subject to
|
11
|
+
the following conditions:
|
12
|
+
|
13
|
+
The above copyright notice and this permission notice shall be
|
14
|
+
included in all copies or substantial portions of the Software.
|
15
|
+
|
16
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
17
|
+
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
18
|
+
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
19
|
+
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
20
|
+
LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
21
|
+
OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
22
|
+
WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,129 @@
|
|
1
|
+
# Rubygems Mini Mirror
|
2
|
+
|
3
|
+
This gem aims at providing a way to mirror only some gems you want from rubygems.
|
4
|
+
|
5
|
+
Create a minigems file
|
6
|
+
|
7
|
+
```ruby
|
8
|
+
# minigems
|
9
|
+
source :gemcutter
|
10
|
+
gem 'rails', ['~> 1.2.0','>= 3.0']
|
11
|
+
gem 'mislav-will_paginate', ['= 2.3.11'], :source => 'http://gems.github.com'
|
12
|
+
# You can also provide a resource file which provide a list of gems
|
13
|
+
# The filetype is guessed by the extension of the file and can be overriden.
|
14
|
+
# :type option can be :yaml, :ruby
|
15
|
+
resource :path => '/Users/hallelujah/repos/mini_gems.yml', :type => 'yaml'
|
16
|
+
```
|
17
|
+
|
18
|
+
The princips
|
19
|
+
------------
|
20
|
+
|
21
|
+
This nice DSL is mainly inspired by Bundler, so it should be familiar to you.
|
22
|
+
|
23
|
+
```ruby
|
24
|
+
# Fetching rails with version 1.2.3 and 2.3.5 with runtime and development dependencies
|
25
|
+
# By default development dependencies are not fetched
|
26
|
+
# *WARNING* fetching dependencies may be a slow process !! Use it with caution.
|
27
|
+
gem 'rails', ['= 1.2.3', '2.3.5'], :development => true
|
28
|
+
|
29
|
+
# If you just want to fetch development dependencies for 1.2.3 version, you need separated definitions :
|
30
|
+
gem 'rails', ['= 1.2.3', '2.3.5'], :development => true
|
31
|
+
gem 'rails', ['2.3.5']
|
32
|
+
|
33
|
+
# To add another resource file :
|
34
|
+
resource :path => 'development_gems.yml'
|
35
|
+
```
|
36
|
+
|
37
|
+
Building your own resource handler is also easy
|
38
|
+
|
39
|
+
```ruby
|
40
|
+
# You can build your own resource handler :
|
41
|
+
# my_resource_handler.rb
|
42
|
+
|
43
|
+
require 'mysql'
|
44
|
+
require 'activerecord'
|
45
|
+
class MyResourceHandler
|
46
|
+
|
47
|
+
# db_gems table structure
|
48
|
+
# name: string
|
49
|
+
# requirements: text
|
50
|
+
# sources: text
|
51
|
+
# development: boolean
|
52
|
+
class DbGem < ActiveRecord::Base
|
53
|
+
serialize :sources, Array
|
54
|
+
serialize :requirements, Array
|
55
|
+
end
|
56
|
+
|
57
|
+
include Gem::MiniMirror::Resource
|
58
|
+
register :type => 'activerecord', :ext => ['.mysql']
|
59
|
+
|
60
|
+
def initialize(runner, options)
|
61
|
+
super
|
62
|
+
@config = YAML.load_file(@path).with_indifferent_access
|
63
|
+
sources(*options[:sources])
|
64
|
+
DbGem.establish_connection @config[:db]
|
65
|
+
end
|
66
|
+
|
67
|
+
def load!
|
68
|
+
super
|
69
|
+
DbGem.all.each do |g|
|
70
|
+
gem g.name, g.requirements, :source => g.sources, :development => g.development?
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
```
|
75
|
+
|
76
|
+
```yaml
|
77
|
+
# database.yml
|
78
|
+
|
79
|
+
sources: :gemcutter
|
80
|
+
db:
|
81
|
+
adapter: mysql
|
82
|
+
database: mini_mirror
|
83
|
+
host: localhost
|
84
|
+
username: root
|
85
|
+
password:
|
86
|
+
```
|
87
|
+
|
88
|
+
```ruby
|
89
|
+
# Usage :
|
90
|
+
require 'rubygems/mini_mirror'
|
91
|
+
require 'my_resource_handler'
|
92
|
+
Gem::MiniMirror::Runner.run :path => 'database.yml', :type => 'activerecord'
|
93
|
+
```
|
94
|
+
|
95
|
+
By gem command
|
96
|
+
--------------
|
97
|
+
|
98
|
+
It provides a gem command
|
99
|
+
|
100
|
+
```bash
|
101
|
+
gem install rubygems-mini_mirror
|
102
|
+
gem mini_mirror help
|
103
|
+
gem mini_mirror -c your_gems_resource.rb -p 5 -m /home/mirrors/rubygems
|
104
|
+
# Generate index
|
105
|
+
gem generate_index --directory=/home/mirrors/rubygems --modern
|
106
|
+
```
|
107
|
+
|
108
|
+
# Thanks
|
109
|
+
|
110
|
+
Thank for the rubygems team for their work on rubygems-mirror which inspired a lot this gem.
|
111
|
+
Some parts of code of this gem are copied from [rubygems-mirror](https://github.com/rubygems/rubygems-mirror) gem.
|
112
|
+
|
113
|
+
|
114
|
+
# Note on Patches/Pull Requests
|
115
|
+
|
116
|
+
* Fork the project.
|
117
|
+
|
118
|
+
* Make your feature addition or bug fix.
|
119
|
+
|
120
|
+
* Add tests for it. This is important so I don’t break it in a future version unintentionally.
|
121
|
+
|
122
|
+
* Commit, do not mess with rakefile, version, or history. (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull)
|
123
|
+
|
124
|
+
* Send me a pull request. Bonus points for topic branches.
|
125
|
+
|
126
|
+
# Copyright
|
127
|
+
|
128
|
+
Copyright © 2011 Ramihajamalala Hery. See LICENSE for details
|
129
|
+
|
data/Rakefile
ADDED
data/TODO.md
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
class Array
|
2
|
+
# Courtesy from activerecord extract_options
|
3
|
+
#
|
4
|
+
# Extracts options from a set of arguments. Removes and returns the last
|
5
|
+
# element in the array if it's a hash, otherwise returns a blank hash.
|
6
|
+
#
|
7
|
+
# def options(*args)
|
8
|
+
# args.extract_options!
|
9
|
+
# end
|
10
|
+
#
|
11
|
+
# options(1, 2) # => {}
|
12
|
+
# options(1, 2, :a => :b) # => {:a=>:b}
|
13
|
+
def extract_options!
|
14
|
+
last.is_a?(::Hash) ? pop : {}
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'rubygems'
|
2
|
+
require "rubygems/mini_mirror/version"
|
3
|
+
require "rubygems/array_ext"
|
4
|
+
require 'yaml'
|
5
|
+
|
6
|
+
module Gem
|
7
|
+
module MiniMirror
|
8
|
+
POOL_SIZE = 10
|
9
|
+
autoload :Pool, 'rubygems/mini_mirror/pool'
|
10
|
+
autoload :Fetcher, 'rubygems/mini_mirror/fetcher'
|
11
|
+
autoload :Runner, 'rubygems/mini_mirror/runner'
|
12
|
+
autoload :Cli, 'rubygems/mini_mirror/cli'
|
13
|
+
autoload :Finder, 'rubygems/mini_mirror/finder'
|
14
|
+
autoload :Resource, 'rubygems/mini_mirror/resource'
|
15
|
+
autoload :Dependency, 'rubygems/mini_mirror/dependency'
|
16
|
+
autoload :ResourceHandler, 'rubygems/mini_mirror/resource_handler'
|
17
|
+
autoload :Resources, 'rubygems/mini_mirror/resources'
|
18
|
+
autoload :Runner, 'rubygems/mini_mirror/runner'
|
19
|
+
|
20
|
+
|
21
|
+
def self.resources_handler
|
22
|
+
ResourceHandler.instance
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
module Gem
|
2
|
+
module MiniMirror
|
3
|
+
module Cli
|
4
|
+
|
5
|
+
# Load default gem configs of the system
|
6
|
+
Gem::ConfigFile.new []
|
7
|
+
|
8
|
+
DEFAULT_SOURCES = {:gemcutter => ['http://gemcutter.org'], :github => ['http://gems.github.com'], :default => Gem.sources}
|
9
|
+
|
10
|
+
def initialize(runner, options)
|
11
|
+
@options = options
|
12
|
+
@runner ||= runner
|
13
|
+
@sources = nil
|
14
|
+
@dependencies ||= []
|
15
|
+
end
|
16
|
+
|
17
|
+
def gem(name, *reqs)
|
18
|
+
options = reqs.extract_options!
|
19
|
+
reqs.flatten!
|
20
|
+
if reqs.empty?
|
21
|
+
reqs = ['>=0']
|
22
|
+
end
|
23
|
+
reqs.each do |req|
|
24
|
+
@dependencies << Gem::MiniMirror::Dependency.new(name, req, options.delete(:source) || @sources, options)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def source(*sources)
|
29
|
+
@sources ||= []
|
30
|
+
@sources |= sources.map{|s| DEFAULT_SOURCES[s] || s }.flatten
|
31
|
+
end
|
32
|
+
|
33
|
+
def resource(options={})
|
34
|
+
errors = catch(:resource_load_error) do
|
35
|
+
@runner.load_resource!(options)
|
36
|
+
nil
|
37
|
+
end
|
38
|
+
if errors
|
39
|
+
handle_error(errors)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
def load!
|
44
|
+
return if @loaded
|
45
|
+
@loaded = true
|
46
|
+
end
|
47
|
+
|
48
|
+
protected
|
49
|
+
def handle_error(errors)
|
50
|
+
# MiniMirror.ui.warn(errors.inspect)
|
51
|
+
warn errors.inspect
|
52
|
+
end
|
53
|
+
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
require 'rubygems/mini_mirror'
|
2
|
+
require 'rubygems/command'
|
3
|
+
require 'yaml'
|
4
|
+
|
5
|
+
class Gem::Commands::MiniMirrorCommand < Gem::Command
|
6
|
+
SUPPORTS_INFO_SIGNAL = Signal.list['INFO']
|
7
|
+
|
8
|
+
def initialize
|
9
|
+
super 'mirror', 'Mirror a gem repository'
|
10
|
+
add_option '-c', '--config-file=FILE', 'the main config file' do |file, options|
|
11
|
+
options[:path] = file
|
12
|
+
end
|
13
|
+
add_option '-t', '--file-type=FILETYPE', 'file type of config file if not guessed' do |t, options|
|
14
|
+
options[:type] = t
|
15
|
+
end
|
16
|
+
add_option '-d', '--cd=DIR', 'directory to cd into before running the command' do |dir,options|
|
17
|
+
options[:cd] = dir
|
18
|
+
end
|
19
|
+
add_option '-r', '--require=lib1,lib2', Array, 'A list of libs to require before running the command' do |libs, options|
|
20
|
+
options[:libs] ||= []
|
21
|
+
options[:libs].push(*libs)
|
22
|
+
end
|
23
|
+
add_option '-m', '--mirror-dir=MIRROR_DIR', 'The mirror root directory : whre to download gem files. Default to <Gem.user_home>/mirror' do |dir,options|
|
24
|
+
options[:gems_dir] = dir
|
25
|
+
end
|
26
|
+
add_option '-p', '--pool-size=POOLSIZE', Integer, 'The number of threads in the pool : fro fetching concurrency. Default 10' do |pool, options|
|
27
|
+
options[:pool_size] = pool
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
def description # :nodoc:
|
32
|
+
<<-EOF
|
33
|
+
The mini_mirror command mirrors remote gem
|
34
|
+
repositories to a local path. Not all the repository of rubygem is mirrored but only those you have chosen with their dependencies
|
35
|
+
|
36
|
+
See https://github.com/hallelujah/rubygems-mini_mirror
|
37
|
+
|
38
|
+
EOF
|
39
|
+
end
|
40
|
+
|
41
|
+
def execute
|
42
|
+
Dir.chdir(options.delete(:cd) || '.') do
|
43
|
+
if options[:requires]
|
44
|
+
options[:requires].each do |lib|
|
45
|
+
require lib
|
46
|
+
end
|
47
|
+
end
|
48
|
+
mirror = Gem::MiniMirror::Runner.new(options)
|
49
|
+
mirror.load_resource! options
|
50
|
+
say "Total gems: #{mirror.gems.size}"
|
51
|
+
num_to_fetch = mirror.gems_to_fetch.size
|
52
|
+
progress = ui.progress_reporter num_to_fetch, "Fetching #{num_to_fetch} gems"
|
53
|
+
trap(:INFO) { puts "Fetched: #{progress.count}/#{num_to_fetch}" } if SUPPORTS_INFO_SIGNAL
|
54
|
+
mirror.update_gems { progress.updated true }
|
55
|
+
num_to_delete = mirror.gems_to_delete.size
|
56
|
+
progress = ui.progress_reporter num_to_delete, "Deleting #{num_to_delete} gems"
|
57
|
+
trap(:INFO) { puts "Fetched: #{progress.count}/#{num_to_delete}" } if SUPPORTS_INFO_SIGNAL
|
58
|
+
mirror.delete_gems { progress.updated true }
|
59
|
+
end
|
60
|
+
|
61
|
+
end
|
62
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
module Gem
|
2
|
+
module MiniMirror
|
3
|
+
class Dependency
|
4
|
+
|
5
|
+
attr_reader :dependency, :sources
|
6
|
+
|
7
|
+
def initialize(name, requirements, *args)
|
8
|
+
options = args.extract_options!
|
9
|
+
srcs = args.empty? ? nil : args.flatten
|
10
|
+
@sources = srcs || Gem.sources
|
11
|
+
@dependency = Gem::Dependency.new(name,requirements)
|
12
|
+
@development = !! options[:development]
|
13
|
+
end
|
14
|
+
|
15
|
+
def development?
|
16
|
+
@development
|
17
|
+
end
|
18
|
+
|
19
|
+
def method_missing(*args)
|
20
|
+
@dependency.send(*args)
|
21
|
+
end
|
22
|
+
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
@@ -0,0 +1,58 @@
|
|
1
|
+
# This code is part of rubygems rubygems-mirror
|
2
|
+
# https://github.com/rubygems/rubygems-mirror
|
3
|
+
require 'net/http'
|
4
|
+
require 'net/http/persistent'
|
5
|
+
|
6
|
+
class Gem::MiniMirror::Fetcher
|
7
|
+
# TODO beef
|
8
|
+
class Error < StandardError; end
|
9
|
+
|
10
|
+
def initialize
|
11
|
+
@http = Net::HTTP::Persistent.new(self.class.name, :ENV)
|
12
|
+
end
|
13
|
+
|
14
|
+
# Fetch a source path under the base uri, and put it in the same or given
|
15
|
+
# destination path under the base path.
|
16
|
+
def fetch(uri, path)
|
17
|
+
modified_time = File.exists?(path) && File.stat(path).mtime.rfc822
|
18
|
+
|
19
|
+
req = Net::HTTP::Get.new URI.parse(uri).path
|
20
|
+
req.add_field 'If-Modified-Since', modified_time if modified_time
|
21
|
+
|
22
|
+
@http.request URI(uri), req do |resp|
|
23
|
+
return handle_response(resp, path)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
# Handle an http response, follow redirects, etc. returns true if a file was
|
28
|
+
# downloaded, false if a 304. Raise Error on unknown responses.
|
29
|
+
def handle_response(resp, path)
|
30
|
+
case resp.code.to_i
|
31
|
+
when 304
|
32
|
+
when 302
|
33
|
+
fetch resp['location'], path
|
34
|
+
when 200
|
35
|
+
write_file(resp, path)
|
36
|
+
when 403, 404
|
37
|
+
warn "#{resp.code} on #{File.basename(path)}"
|
38
|
+
else
|
39
|
+
raise Error, "unexpected response #{resp.inspect}"
|
40
|
+
end
|
41
|
+
# TODO rescue http errors and reraise cleanly
|
42
|
+
end
|
43
|
+
|
44
|
+
# Efficiently writes an http response object to a particular path. If there
|
45
|
+
# is an error, it will remove the target file.
|
46
|
+
def write_file(resp, path)
|
47
|
+
FileUtils.mkdir_p File.dirname(path)
|
48
|
+
File.open(path, 'wb') do |output|
|
49
|
+
resp.read_body { |chunk| output << chunk }
|
50
|
+
end
|
51
|
+
true
|
52
|
+
ensure
|
53
|
+
# cleanup incomplete files, rescue perm errors etc, they're being
|
54
|
+
# raised already.
|
55
|
+
File.delete(path) rescue nil if $!
|
56
|
+
end
|
57
|
+
|
58
|
+
end
|
@@ -0,0 +1,56 @@
|
|
1
|
+
module Gem
|
2
|
+
module MiniMirror
|
3
|
+
module Finder
|
4
|
+
|
5
|
+
def initialize(options = {})
|
6
|
+
@spec_fetcher = Gem::SpecFetcher.fetcher
|
7
|
+
super()
|
8
|
+
end
|
9
|
+
|
10
|
+
def find_all_specs
|
11
|
+
@dependencies.each do |dep|
|
12
|
+
with_sources dep.sources do
|
13
|
+
found, errors = @spec_fetcher.fetch_with_errors dep, true, false
|
14
|
+
found.each do |spec,source_uri|
|
15
|
+
next if is_in_specs?(spec)
|
16
|
+
add_to_specs(spec,source_uri)
|
17
|
+
add_to_deps(*spec.runtime_dependencies)
|
18
|
+
add_to_deps(*spec.development_dependencies) if dep.development?
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def add_to_specs(spec, source_uri)
|
25
|
+
@specs.push([spec, source_uri])
|
26
|
+
@specs_list[spec.platform.to_s][spec.name.to_s][spec.version.to_s] = true
|
27
|
+
end
|
28
|
+
|
29
|
+
def is_in_specs?(spec)
|
30
|
+
@specs_list.key?(spec.platform.to_s) && @specs_list[spec.platform.to_s].key?(spec.name.to_s) && @specs_list[spec.platform.to_s][spec.name.to_s].has_key?(spec.version.to_s)
|
31
|
+
end
|
32
|
+
|
33
|
+
def add_to_deps(*deps)
|
34
|
+
deps.each do |dep|
|
35
|
+
next if is_in_deps?(dep)
|
36
|
+
dep = Gem::MiniMirror::Dependency.new(dep.name, dep.requirement,dep.respond_to?(:sources) ? dep.sources : Gem.sources, {:development => dep.respond_to?(:development?) ? dep.development? : false})
|
37
|
+
@dependencies_list[dep.name.to_s][dep.requirement.to_s] = true
|
38
|
+
@dependencies.push(dep)
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def is_in_deps?(dep)
|
43
|
+
@dependencies_list[dep.name.to_s][dep.requirement.to_s] == true
|
44
|
+
end
|
45
|
+
|
46
|
+
|
47
|
+
def with_sources(srcs,&block)
|
48
|
+
before_sources = Gem.sources
|
49
|
+
Gem::sources= srcs
|
50
|
+
block.call
|
51
|
+
Gem.sources= before_sources
|
52
|
+
end
|
53
|
+
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'thread'
|
2
|
+
class Gem::MiniMirror::Pool
|
3
|
+
def initialize(size)
|
4
|
+
@size = size
|
5
|
+
@queue = Queue.new
|
6
|
+
end
|
7
|
+
|
8
|
+
def job(&blk)
|
9
|
+
@queue << blk
|
10
|
+
end
|
11
|
+
|
12
|
+
def run_til_done
|
13
|
+
threads = Array.new(@size) do
|
14
|
+
Thread.new { @queue.pop.call while true }
|
15
|
+
end
|
16
|
+
until @queue.empty? && @queue.num_waiting == @size
|
17
|
+
threads.each { |t| t.join(0.1) }
|
18
|
+
end
|
19
|
+
threads.each { |t| t.kill }
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module Gem
|
2
|
+
module MiniMirror
|
3
|
+
|
4
|
+
module Resource
|
5
|
+
|
6
|
+
def self.included(base)
|
7
|
+
base.class_eval do
|
8
|
+
attr_accessor :sources
|
9
|
+
attr_reader :dependencies
|
10
|
+
include Gem::MiniMirror::Cli
|
11
|
+
extend ClassMethods
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
module ClassMethods
|
16
|
+
def register(options={})
|
17
|
+
Gem::MiniMirror.resources_handler.add(self, options[:type], options[:ext])
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def tag
|
22
|
+
@options.to_s
|
23
|
+
end
|
24
|
+
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,62 @@
|
|
1
|
+
require 'singleton'
|
2
|
+
module Gem
|
3
|
+
module MiniMirror
|
4
|
+
class ResourceHandler
|
5
|
+
class AlreadyRegistered < Exception
|
6
|
+
def initialize(type)
|
7
|
+
@type = type
|
8
|
+
end
|
9
|
+
def message
|
10
|
+
"already registered #@type"
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
class HandlerNotFound < Exception
|
15
|
+
def initialize(options = {})
|
16
|
+
@type = options[:type]
|
17
|
+
@extension = options[:ext]
|
18
|
+
end
|
19
|
+
|
20
|
+
def message
|
21
|
+
"could not find handler for file extension '#@extension'" +
|
22
|
+
@type.to_s.empty? ? "" : " or for file type #@type"
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
include Singleton
|
27
|
+
|
28
|
+
def initialize
|
29
|
+
@handlers = {}
|
30
|
+
@handlers_by_file_extension = {}
|
31
|
+
end
|
32
|
+
|
33
|
+
def add(klass, type, exts = [])
|
34
|
+
raise AlreadyRegistered, type if @handlers.key?(type.to_s)
|
35
|
+
@handlers[type.to_s] = klass
|
36
|
+
exts.each do |ext|
|
37
|
+
@handlers_by_file_extension[ext.to_s] = klass unless ext.to_s.empty?
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def find_handler_by_extension(ext)
|
42
|
+
@handlers_by_file_extension[ext.to_s]
|
43
|
+
end
|
44
|
+
|
45
|
+
def find_handler_by_type(type)
|
46
|
+
@handlers[type.to_s]
|
47
|
+
end
|
48
|
+
|
49
|
+
def find(options = {})
|
50
|
+
unless options.key?(:ext)
|
51
|
+
if path = options[:path]
|
52
|
+
options[:ext] = File.extname(path)
|
53
|
+
end
|
54
|
+
end
|
55
|
+
handler = find_handler_by_type(options[:type]) || find_handler_by_extension(options[:ext])
|
56
|
+
raise HandlerNotFound, options unless handler
|
57
|
+
handler
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
end
|
62
|
+
end
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module Gem
|
2
|
+
module MiniMirror
|
3
|
+
module Resources
|
4
|
+
class BaseFile
|
5
|
+
include Gem::MiniMirror::Resource
|
6
|
+
|
7
|
+
def initialize(runner, options)
|
8
|
+
path = File.expand_path(options[:path])
|
9
|
+
throw :resource_load_error, :path => path unless File.exist?(path)
|
10
|
+
@path = path
|
11
|
+
super
|
12
|
+
end
|
13
|
+
|
14
|
+
def tag
|
15
|
+
'path-' + @path
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'yaml'
|
2
|
+
module Gem
|
3
|
+
module MiniMirror
|
4
|
+
module Resources
|
5
|
+
class Yaml < BaseFile
|
6
|
+
register(:type => 'yaml', :ext => ['.yml', '.yaml'])
|
7
|
+
|
8
|
+
def load!
|
9
|
+
super
|
10
|
+
config = YAML.load_file(@path)
|
11
|
+
config.each do |instructs|
|
12
|
+
instructs.each do |k, args|
|
13
|
+
send(k, *args)
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
17
|
+
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,109 @@
|
|
1
|
+
require 'rubygems/mini_mirror/resources'
|
2
|
+
module Gem
|
3
|
+
module MiniMirror
|
4
|
+
class Runner
|
5
|
+
|
6
|
+
include Gem::MiniMirror::Finder
|
7
|
+
attr_reader :specs, :resource_handler
|
8
|
+
DEFAULT_GEMS_DIR = File.join(Gem.user_home, '.gem', 'mirror')
|
9
|
+
|
10
|
+
def initialize(options = {})
|
11
|
+
@resource_handler = ResourceHandler.instance
|
12
|
+
@dependencies = []
|
13
|
+
@specs_list = Hash.new{|h,k| h[k] = Hash.new{ |h1,k1| h1[k1] = Hash.new}}
|
14
|
+
@specs = []
|
15
|
+
@dependencies_list = Hash.new{|h,k| h[k] = Hash.new}
|
16
|
+
@to_fetch = {}
|
17
|
+
@resources = []
|
18
|
+
@resources_signatures = {}
|
19
|
+
@pool = Gem::MiniMirror::Pool.new(options[:pool_size] || Gem::MiniMirror::POOL_SIZE)
|
20
|
+
@gems_dir = options[:gems_dir] || DEFAULT_GEMS_DIR
|
21
|
+
@fetcher = Gem::MiniMirror::Fetcher.new
|
22
|
+
super
|
23
|
+
end
|
24
|
+
|
25
|
+
def load_resource!(options)
|
26
|
+
klass = @resource_handler.find(options)
|
27
|
+
resource = klass.new(self, options)
|
28
|
+
unless @resources_signatures[resource.tag]
|
29
|
+
resource.load!
|
30
|
+
@resources << resource
|
31
|
+
@resources_signatures[resource.tag] = true
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
def from(*args)
|
36
|
+
File.join(*args)
|
37
|
+
end
|
38
|
+
|
39
|
+
def to(*args)
|
40
|
+
File.join(@gems_dir,*args)
|
41
|
+
end
|
42
|
+
|
43
|
+
def existing_gems
|
44
|
+
Dir[to('gems','.*gem')].entries.map{|f| File.basename(f)}
|
45
|
+
end
|
46
|
+
|
47
|
+
def gems_with_sources
|
48
|
+
load_all!
|
49
|
+
@gems_with_sources ||= Hash[*specs.map{|s,src| [s.full_name + '.gem', src] }.flatten]
|
50
|
+
end
|
51
|
+
|
52
|
+
def gems
|
53
|
+
@gems ||= gems_with_sources.keys
|
54
|
+
end
|
55
|
+
|
56
|
+
def gems_to_fetch
|
57
|
+
gems - existing_gems
|
58
|
+
end
|
59
|
+
|
60
|
+
def gems_to_delete
|
61
|
+
existing_gems - gems
|
62
|
+
end
|
63
|
+
|
64
|
+
def update_gems
|
65
|
+
gems_to_fetch.each do |g|
|
66
|
+
@pool.job do
|
67
|
+
@fetcher.fetch(from(gems_with_sources[g],'gems',g),to('gems',g))
|
68
|
+
yield
|
69
|
+
end
|
70
|
+
end
|
71
|
+
@pool.run_til_done
|
72
|
+
end
|
73
|
+
|
74
|
+
def delete_gems
|
75
|
+
gems_to_delete.each do |g|
|
76
|
+
@pool.job do
|
77
|
+
File.delete(to('gems',g))
|
78
|
+
yield
|
79
|
+
end
|
80
|
+
end
|
81
|
+
@pool.run_til_done
|
82
|
+
end
|
83
|
+
|
84
|
+
def update
|
85
|
+
update_gems
|
86
|
+
delete_gems
|
87
|
+
end
|
88
|
+
|
89
|
+
def self.run(file, options = {})
|
90
|
+
runner = new(options)
|
91
|
+
runner.load_resource! :path => file, :type => 'ruby'
|
92
|
+
runner.update
|
93
|
+
end
|
94
|
+
|
95
|
+
protected
|
96
|
+
def load_all!
|
97
|
+
return if @loaded
|
98
|
+
@resources.each do |r|
|
99
|
+
add_to_deps(*r.dependencies)
|
100
|
+
end
|
101
|
+
@loaded = true
|
102
|
+
find_all_specs
|
103
|
+
end
|
104
|
+
|
105
|
+
|
106
|
+
|
107
|
+
end
|
108
|
+
end
|
109
|
+
end
|
@@ -0,0 +1,41 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
$:.push File.expand_path("../lib", __FILE__)
|
3
|
+
require "rubygems/mini_mirror/version"
|
4
|
+
|
5
|
+
Gem::Specification.new do |s|
|
6
|
+
s.name = "rubygems-mini_mirror"
|
7
|
+
s.version = Gem::MiniMirror::VERSION
|
8
|
+
s.authors = ["Ramihajamalala Hery"]
|
9
|
+
s.email = ["hery@rails-royce.org"]
|
10
|
+
s.homepage = ""
|
11
|
+
s.summary = %q{A rubygems mini mirror}
|
12
|
+
s.description = %q{
|
13
|
+
Mirror some version of gems with Gem::Version DSL
|
14
|
+
Create a mini_gem file and add this to it :
|
15
|
+
source :gemcutter
|
16
|
+
gem 'rails', ['~> 1.2.0', '>= 3.0']
|
17
|
+
|
18
|
+
or
|
19
|
+
source :gemcutter
|
20
|
+
resource :path => '/your_path/mini_gem.yml'
|
21
|
+
|
22
|
+
# /your_path/mini_gem.yml
|
23
|
+
|
24
|
+
gem:
|
25
|
+
- rails:
|
26
|
+
-
|
27
|
+
- '~> 1.2.0'
|
28
|
+
- '>= 3.0'
|
29
|
+
|
30
|
+
It will solve the dependencies for you so you don't have to write an exhaustive list of the gems you want to mirror
|
31
|
+
}
|
32
|
+
|
33
|
+
s.rubyforge_project = "rubygems-mini_mirror"
|
34
|
+
s.add_runtime_dependency 'net-http-persistent', '>=1.2.5'
|
35
|
+
s.add_development_dependency 'rake'
|
36
|
+
|
37
|
+
s.files = `git ls-files`.split("\n")
|
38
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
39
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
40
|
+
s.require_paths = ["lib"]
|
41
|
+
end
|
@@ -0,0 +1,140 @@
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
2
|
+
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
|
3
|
+
<plist version="1.0">
|
4
|
+
<dict>
|
5
|
+
<key>currentDocument</key>
|
6
|
+
<string>lib/rubygems/mini_mirror/cli.rb</string>
|
7
|
+
<key>documents</key>
|
8
|
+
<array>
|
9
|
+
<dict>
|
10
|
+
<key>expanded</key>
|
11
|
+
<true/>
|
12
|
+
<key>name</key>
|
13
|
+
<string>rubygems-mini_mirror</string>
|
14
|
+
<key>regexFolderFilter</key>
|
15
|
+
<string>!.*/(\.[^/]*|CVS|_darcs|_MTN|\{arch\}|blib|.*~\.nib|.*\.(framework|app|pbproj|pbxproj|xcode(proj)?|bundle))$</string>
|
16
|
+
<key>sourceDirectory</key>
|
17
|
+
<string></string>
|
18
|
+
</dict>
|
19
|
+
</array>
|
20
|
+
<key>fileHierarchyDrawerWidth</key>
|
21
|
+
<integer>262</integer>
|
22
|
+
<key>metaData</key>
|
23
|
+
<dict>
|
24
|
+
<key>lib/rubygems/mini_mirror.rb</key>
|
25
|
+
<dict>
|
26
|
+
<key>caret</key>
|
27
|
+
<dict>
|
28
|
+
<key>column</key>
|
29
|
+
<integer>30</integer>
|
30
|
+
<key>line</key>
|
31
|
+
<integer>11</integer>
|
32
|
+
</dict>
|
33
|
+
<key>firstVisibleColumn</key>
|
34
|
+
<integer>0</integer>
|
35
|
+
<key>firstVisibleLine</key>
|
36
|
+
<integer>0</integer>
|
37
|
+
</dict>
|
38
|
+
<key>lib/rubygems/mini_mirror/cli.rb</key>
|
39
|
+
<dict>
|
40
|
+
<key>caret</key>
|
41
|
+
<dict>
|
42
|
+
<key>column</key>
|
43
|
+
<integer>28</integer>
|
44
|
+
<key>line</key>
|
45
|
+
<integer>25</integer>
|
46
|
+
</dict>
|
47
|
+
<key>columnSelection</key>
|
48
|
+
<false/>
|
49
|
+
<key>firstVisibleColumn</key>
|
50
|
+
<integer>0</integer>
|
51
|
+
<key>firstVisibleLine</key>
|
52
|
+
<integer>0</integer>
|
53
|
+
<key>selectFrom</key>
|
54
|
+
<dict>
|
55
|
+
<key>column</key>
|
56
|
+
<integer>24</integer>
|
57
|
+
<key>line</key>
|
58
|
+
<integer>25</integer>
|
59
|
+
</dict>
|
60
|
+
<key>selectTo</key>
|
61
|
+
<dict>
|
62
|
+
<key>column</key>
|
63
|
+
<integer>32</integer>
|
64
|
+
<key>line</key>
|
65
|
+
<integer>25</integer>
|
66
|
+
</dict>
|
67
|
+
</dict>
|
68
|
+
<key>lib/rubygems/mini_mirror/resource.rb</key>
|
69
|
+
<dict>
|
70
|
+
<key>caret</key>
|
71
|
+
<dict>
|
72
|
+
<key>column</key>
|
73
|
+
<integer>78</integer>
|
74
|
+
<key>line</key>
|
75
|
+
<integer>16</integer>
|
76
|
+
</dict>
|
77
|
+
<key>firstVisibleColumn</key>
|
78
|
+
<integer>0</integer>
|
79
|
+
<key>firstVisibleLine</key>
|
80
|
+
<integer>0</integer>
|
81
|
+
</dict>
|
82
|
+
<key>lib/rubygems/mini_mirror/resources/ruby_file.rb</key>
|
83
|
+
<dict>
|
84
|
+
<key>caret</key>
|
85
|
+
<dict>
|
86
|
+
<key>column</key>
|
87
|
+
<integer>54</integer>
|
88
|
+
<key>line</key>
|
89
|
+
<integer>15</integer>
|
90
|
+
</dict>
|
91
|
+
<key>firstVisibleColumn</key>
|
92
|
+
<integer>0</integer>
|
93
|
+
<key>firstVisibleLine</key>
|
94
|
+
<integer>0</integer>
|
95
|
+
</dict>
|
96
|
+
<key>lib/rubygems/mini_mirror/runner.rb</key>
|
97
|
+
<dict>
|
98
|
+
<key>caret</key>
|
99
|
+
<dict>
|
100
|
+
<key>column</key>
|
101
|
+
<integer>16</integer>
|
102
|
+
<key>line</key>
|
103
|
+
<integer>16</integer>
|
104
|
+
</dict>
|
105
|
+
<key>columnSelection</key>
|
106
|
+
<false/>
|
107
|
+
<key>firstVisibleColumn</key>
|
108
|
+
<integer>0</integer>
|
109
|
+
<key>firstVisibleLine</key>
|
110
|
+
<integer>0</integer>
|
111
|
+
<key>selectFrom</key>
|
112
|
+
<dict>
|
113
|
+
<key>column</key>
|
114
|
+
<integer>10</integer>
|
115
|
+
<key>line</key>
|
116
|
+
<integer>16</integer>
|
117
|
+
</dict>
|
118
|
+
<key>selectTo</key>
|
119
|
+
<dict>
|
120
|
+
<key>column</key>
|
121
|
+
<integer>23</integer>
|
122
|
+
<key>line</key>
|
123
|
+
<integer>16</integer>
|
124
|
+
</dict>
|
125
|
+
</dict>
|
126
|
+
</dict>
|
127
|
+
<key>openDocuments</key>
|
128
|
+
<array>
|
129
|
+
<string>lib/rubygems/mini_mirror/resources/ruby_file.rb</string>
|
130
|
+
<string>lib/rubygems/mini_mirror/runner.rb</string>
|
131
|
+
<string>lib/rubygems/mini_mirror/resource.rb</string>
|
132
|
+
<string>lib/rubygems/mini_mirror.rb</string>
|
133
|
+
<string>lib/rubygems/mini_mirror/cli.rb</string>
|
134
|
+
</array>
|
135
|
+
<key>showFileHierarchyDrawer</key>
|
136
|
+
<true/>
|
137
|
+
<key>windowFrame</key>
|
138
|
+
<string>{{2192, -120}, {1648, 1200}}</string>
|
139
|
+
</dict>
|
140
|
+
</plist>
|
metadata
ADDED
@@ -0,0 +1,121 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: rubygems-mini_mirror
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
hash: 23
|
5
|
+
prerelease: false
|
6
|
+
segments:
|
7
|
+
- 1
|
8
|
+
- 0
|
9
|
+
- 0
|
10
|
+
version: 1.0.0
|
11
|
+
platform: ruby
|
12
|
+
authors:
|
13
|
+
- Ramihajamalala Hery
|
14
|
+
autorequire:
|
15
|
+
bindir: bin
|
16
|
+
cert_chain: []
|
17
|
+
|
18
|
+
date: 2011-09-26 00:00:00 +02:00
|
19
|
+
default_executable:
|
20
|
+
dependencies:
|
21
|
+
- !ruby/object:Gem::Dependency
|
22
|
+
name: net-http-persistent
|
23
|
+
prerelease: false
|
24
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
25
|
+
none: false
|
26
|
+
requirements:
|
27
|
+
- - ">="
|
28
|
+
- !ruby/object:Gem::Version
|
29
|
+
hash: 21
|
30
|
+
segments:
|
31
|
+
- 1
|
32
|
+
- 2
|
33
|
+
- 5
|
34
|
+
version: 1.2.5
|
35
|
+
type: :runtime
|
36
|
+
version_requirements: *id001
|
37
|
+
- !ruby/object:Gem::Dependency
|
38
|
+
name: rake
|
39
|
+
prerelease: false
|
40
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
41
|
+
none: false
|
42
|
+
requirements:
|
43
|
+
- - ">="
|
44
|
+
- !ruby/object:Gem::Version
|
45
|
+
hash: 3
|
46
|
+
segments:
|
47
|
+
- 0
|
48
|
+
version: "0"
|
49
|
+
type: :development
|
50
|
+
version_requirements: *id002
|
51
|
+
description: "\n Mirror some version of gems with Gem::Version DSL\n Create a mini_gem file and add this to it :\n source :gemcutter\n gem 'rails', ['~> 1.2.0', '>= 3.0']\n\n or\n source :gemcutter\n resource :path => '/your_path/mini_gem.yml'\n\n # /your_path/mini_gem.yml\n\n gem:\n - rails:\n -\n - '~> 1.2.0'\n - '>= 3.0'\n\n It will solve the dependencies for you so you don't have to write an exhaustive list of the gems you want to mirror\n "
|
52
|
+
email:
|
53
|
+
- hery@rails-royce.org
|
54
|
+
executables: []
|
55
|
+
|
56
|
+
extensions: []
|
57
|
+
|
58
|
+
extra_rdoc_files: []
|
59
|
+
|
60
|
+
files:
|
61
|
+
- .gitignore
|
62
|
+
- Gemfile
|
63
|
+
- LICENSE
|
64
|
+
- README.md
|
65
|
+
- Rakefile
|
66
|
+
- TODO.md
|
67
|
+
- lib/rubygems/array_ext.rb
|
68
|
+
- lib/rubygems/mini_mirror.rb
|
69
|
+
- lib/rubygems/mini_mirror/cli.rb
|
70
|
+
- lib/rubygems/mini_mirror/command.rb
|
71
|
+
- lib/rubygems/mini_mirror/dependency.rb
|
72
|
+
- lib/rubygems/mini_mirror/fetcher.rb
|
73
|
+
- lib/rubygems/mini_mirror/finder.rb
|
74
|
+
- lib/rubygems/mini_mirror/pool.rb
|
75
|
+
- lib/rubygems/mini_mirror/resource.rb
|
76
|
+
- lib/rubygems/mini_mirror/resource_handler.rb
|
77
|
+
- lib/rubygems/mini_mirror/resources.rb
|
78
|
+
- lib/rubygems/mini_mirror/resources/base_file.rb
|
79
|
+
- lib/rubygems/mini_mirror/resources/ruby_file.rb
|
80
|
+
- lib/rubygems/mini_mirror/resources/yaml.rb
|
81
|
+
- lib/rubygems/mini_mirror/runner.rb
|
82
|
+
- lib/rubygems/mini_mirror/version.rb
|
83
|
+
- lib/rubygems_plugin.rb
|
84
|
+
- rubygems-mini_mirror.gemspec
|
85
|
+
- rubygems-mini_mirror.tmproj
|
86
|
+
has_rdoc: true
|
87
|
+
homepage: ""
|
88
|
+
licenses: []
|
89
|
+
|
90
|
+
post_install_message:
|
91
|
+
rdoc_options: []
|
92
|
+
|
93
|
+
require_paths:
|
94
|
+
- lib
|
95
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
96
|
+
none: false
|
97
|
+
requirements:
|
98
|
+
- - ">="
|
99
|
+
- !ruby/object:Gem::Version
|
100
|
+
hash: 3
|
101
|
+
segments:
|
102
|
+
- 0
|
103
|
+
version: "0"
|
104
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
105
|
+
none: false
|
106
|
+
requirements:
|
107
|
+
- - ">="
|
108
|
+
- !ruby/object:Gem::Version
|
109
|
+
hash: 3
|
110
|
+
segments:
|
111
|
+
- 0
|
112
|
+
version: "0"
|
113
|
+
requirements: []
|
114
|
+
|
115
|
+
rubyforge_project: rubygems-mini_mirror
|
116
|
+
rubygems_version: 1.3.7
|
117
|
+
signing_key:
|
118
|
+
specification_version: 3
|
119
|
+
summary: A rubygems mini mirror
|
120
|
+
test_files: []
|
121
|
+
|