allgems 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG +0 -0
- data/README.rdoc +16 -0
- data/allgems.gemspec +24 -0
- data/bin/allgems +215 -0
- data/config.ru +11 -0
- data/lib/allgems.rb +69 -0
- data/lib/allgems/App.rb +92 -0
- data/lib/allgems/Configurator.rb +0 -0
- data/lib/allgems/GemWorker.rb +229 -0
- data/lib/allgems/IndexBuilder.rb +17 -0
- data/lib/allgems/Indexer.rb +23 -0
- data/lib/allgems/Runner.rb +59 -0
- data/lib/allgems/Specer.rb +12 -0
- data/lib/allgems/ViewHelpers.rb +48 -0
- data/lib/allgems/hanna_hack.rb +82 -0
- data/lib/allgems/migrations/001_initialize.rb +39 -0
- data/public/images/edit.png +0 -0
- data/public/images/folder.png +0 -0
- data/public/images/git.gif +0 -0
- data/public/images/page.png +0 -0
- data/public/images/page_white_text.png +0 -0
- data/public/images/rubygems-125x125t.png +0 -0
- data/public/javascripts/base.js +212 -0
- data/public/javascripts/gembox.js +77 -0
- data/public/javascripts/jquery.form.js +632 -0
- data/public/javascripts/jquery.js +4241 -0
- data/public/javascripts/jquery.metadata.js +121 -0
- data/public/javascripts/jquery.ui.js +273 -0
- data/public/swf/clippy.swf +0 -0
- data/views/doc.haml +17 -0
- data/views/file_tree.haml +9 -0
- data/views/gem.haml +45 -0
- data/views/gembox.sass +188 -0
- data/views/gems_columns.haml +24 -0
- data/views/gems_header.haml +15 -0
- data/views/gems_table.haml +28 -0
- data/views/index.haml +1 -0
- data/views/layout.haml +59 -0
- data/views/load.haml +16 -0
- data/views/no_results.haml +3 -0
- metadata +185 -0
data/CHANGELOG
ADDED
File without changes
|
data/README.rdoc
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
= AllGems
|
2
|
+
|
3
|
+
== Description
|
4
|
+
|
5
|
+
AllGems is a simple project with a hefty goal: Provide easy access to documentation for all RubyGem files.
|
6
|
+
|
7
|
+
== How?
|
8
|
+
|
9
|
+
==== Backend
|
10
|
+
|
11
|
+
AllGems runs a small daemon that monitors the index of any sources registered with the locally installed RubyGems. This daemon automatically retrieves, unpacks, and generates documentation on libraries as they are released.
|
12
|
+
|
13
|
+
==== Frontend
|
14
|
+
|
15
|
+
AllGems uses a heavily modified version of Gembox to allow access to available documentation.
|
16
|
+
|
data/allgems.gemspec
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
spec = Gem::Specification.new do |s|
|
2
|
+
s.name = 'allgems'
|
3
|
+
s.author = %q(spox)
|
4
|
+
s.email = %q(spox@modspox.com)
|
5
|
+
s.version = '0.0.1'
|
6
|
+
s.summary = %q(Tools to document the world)
|
7
|
+
s.platform = Gem::Platform::RUBY
|
8
|
+
s.files = Dir['**/*']
|
9
|
+
s.rdoc_options = %w(--title AllGems --main README.rdoc --line-numbers)
|
10
|
+
s.extra_rdoc_files = %w(README.rdoc CHANGELOG)
|
11
|
+
s.require_paths = %w(lib)
|
12
|
+
s.executables = %w(allgems)
|
13
|
+
s.required_ruby_version = '>= 1.8.6'
|
14
|
+
s.homepage = %q(http://github.com/spox/allgems)
|
15
|
+
s.description = 'AllGems is a tool to provide comprehensive gem documentation for an entire index'
|
16
|
+
s.add_dependency 'sequel'
|
17
|
+
s.add_dependency 'ActionPool'
|
18
|
+
s.add_dependency 'ActionTimer'
|
19
|
+
s.add_dependency 'haml'
|
20
|
+
s.add_dependency 'rdoc'
|
21
|
+
s.add_dependency 'sdoc'
|
22
|
+
s.add_dependency 'hanna'
|
23
|
+
s.add_dependency 'nokogiri'
|
24
|
+
end
|
data/bin/allgems
ADDED
@@ -0,0 +1,215 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
# encoding: utf-8
|
3
|
+
|
4
|
+
SUCCESS = 0
|
5
|
+
CONFIGURATION_ERROR = 1
|
6
|
+
|
7
|
+
def display_help
|
8
|
+
$stdout.puts "\t\tAllGems - Document Everything
|
9
|
+
usage: allgems [opts]
|
10
|
+
--config -c:\t\t\t\tOutput apache configuration
|
11
|
+
--notall -n:\t\t\t\tDon't fetch all versions
|
12
|
+
--daemonize -D:\t\t\t\tRun as a daemon
|
13
|
+
--datadir -d /path:\t\t\tData directory for documents/gems (defaults to /tmp/allgems)
|
14
|
+
--dbfile -b /path/file.db\t\tPath to sqlite file (defaults datadir/allgems.db)
|
15
|
+
--formatter -f rdoc[,hanna,sdoc]:\t\tRDoc formatter (defaults to rdoc (which is darkfish))
|
16
|
+
--interval -i \\d+:\t\t\tSeconds to wait before updating documents (defaults to 1 hour)
|
17
|
+
--runners -r \\d+:\t\t\tNumber of threads to use (defaults to 10)
|
18
|
+
--log -L [/path/file]:\t\t\tTurn logging on (no argument defaults to STDOUT)
|
19
|
+
--verbosity -V [(debug|info|warn|fatal)]:\tLogging level (no argument defaults to info)
|
20
|
+
--help -h:\t\t\t\tDisplay this help text"
|
21
|
+
end
|
22
|
+
|
23
|
+
# load in our libraries
|
24
|
+
['logger', 'fileutils', 'getoptlong', 'allgems', 'sequel'].each{|f|require f}
|
25
|
+
|
26
|
+
# figure out what system we are on
|
27
|
+
require 'rbconfig'
|
28
|
+
ON_WINDOWS = Config::CONFIG['host_os'] =~ /mswin|mingw/
|
29
|
+
|
30
|
+
# set default values
|
31
|
+
config_output = false
|
32
|
+
daemonize = false
|
33
|
+
datadir = '/tmp/allgems'
|
34
|
+
formatter = nil
|
35
|
+
interval = 3600 # default to hourly checks
|
36
|
+
runners = 10 # default to 10 threads
|
37
|
+
dbfile = nil
|
38
|
+
logto = nil
|
39
|
+
level = Logger::INFO
|
40
|
+
all = true
|
41
|
+
|
42
|
+
# figure out what we are doing
|
43
|
+
opts = GetoptLong.new(
|
44
|
+
['--daemonize', '-D', GetoptLong::NO_ARGUMENT],
|
45
|
+
['--datadir', '-d', GetoptLong::REQUIRED_ARGUMENT],
|
46
|
+
['--dbfile', '-b', GetoptLong::REQUIRED_ARGUMENT],
|
47
|
+
['--formatter', '-f', GetoptLong::REQUIRED_ARGUMENT],
|
48
|
+
['--interval', '-i', GetoptLong::REQUIRED_ARGUMENT],
|
49
|
+
['--runners', '-r', GetoptLong::REQUIRED_ARGUMENT],
|
50
|
+
['--version', '-v', GetoptLong::NO_ARGUMENT],
|
51
|
+
['--log', '-L', GetoptLong::OPTIONAL_ARGUMENT],
|
52
|
+
['--verbosity', '-V', GetoptLong::OPTIONAL_ARGUMENT],
|
53
|
+
['--help', '-h', GetoptLong::NO_ARGUMENT],
|
54
|
+
['--config', '-c', GetoptLong::NO_ARGUMENT],
|
55
|
+
['--notall', '-n', GetoptLong::NO_ARGUMENT]
|
56
|
+
)
|
57
|
+
|
58
|
+
opts.each do |opt, arg|
|
59
|
+
case opt
|
60
|
+
when '--daemonize'
|
61
|
+
daemonize = true
|
62
|
+
when '--datadir'
|
63
|
+
datadir = arg
|
64
|
+
when '--formatter'
|
65
|
+
formatter = arg
|
66
|
+
when '--interval'
|
67
|
+
interval = arg.to_i
|
68
|
+
when '--runners'
|
69
|
+
runners = arg.to_i
|
70
|
+
when '--dbfile'
|
71
|
+
dbfile = arg
|
72
|
+
when '--version'
|
73
|
+
puts "VERSION INFO"
|
74
|
+
when '--log'
|
75
|
+
logto = arg.empty? ? $stdout: arg
|
76
|
+
when '--verbosity'
|
77
|
+
case arg
|
78
|
+
when 'debug'
|
79
|
+
level = Logger::DEBUG
|
80
|
+
when 'info'
|
81
|
+
level = Logger::INFO
|
82
|
+
when 'warn'
|
83
|
+
level = Logger::WARN
|
84
|
+
when 'error'
|
85
|
+
level = Logger::ERROR
|
86
|
+
when 'fatal'
|
87
|
+
level = Logger::FATAL
|
88
|
+
else
|
89
|
+
level = Logger::INFO
|
90
|
+
end
|
91
|
+
when '--notall'
|
92
|
+
all = false
|
93
|
+
when '--config'
|
94
|
+
config_output = true
|
95
|
+
when '--help'
|
96
|
+
display_help
|
97
|
+
exit SUCCESS
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
dbfile = "#{datadir}/allgems.db" if dbfile.nil?
|
102
|
+
interval = nil unless daemonize
|
103
|
+
|
104
|
+
# yell at user for bad values
|
105
|
+
if(interval && interval <= 0)
|
106
|
+
$stderr.puts "ERROR: interval value must be greater than 1"
|
107
|
+
exit CONFIGURATION_ERROR
|
108
|
+
end
|
109
|
+
if(runners <= 0)
|
110
|
+
$stderr.puts "ERROR: runners value must be greater than 1"
|
111
|
+
exit CONFIGURATION_ERROR
|
112
|
+
end
|
113
|
+
unless(File.writable?(datadir))
|
114
|
+
created = false
|
115
|
+
#check if we can/should create it
|
116
|
+
unless(File.exists?(datadir))
|
117
|
+
begin
|
118
|
+
FileUtils.mkdir_p datadir
|
119
|
+
created = true
|
120
|
+
rescue SystemCallError
|
121
|
+
created = false
|
122
|
+
end
|
123
|
+
end
|
124
|
+
datadir = File.expand_path(datadir)
|
125
|
+
unless(created)
|
126
|
+
$stderr.puts "ERROR: data directory is not writable. (#{datadir})"
|
127
|
+
exit CONFIGURATION_ERROR
|
128
|
+
end
|
129
|
+
end
|
130
|
+
unless(File.exists?(dbfile))
|
131
|
+
begin
|
132
|
+
FileUtils.touch dbfile
|
133
|
+
rescue SystemCallError
|
134
|
+
end
|
135
|
+
end
|
136
|
+
unless(File.writable?(dbfile))
|
137
|
+
$stderr.puts "ERROR: database file is not writable. (#{dbfile})"
|
138
|
+
exit CONFIGURATION_ERROR
|
139
|
+
end
|
140
|
+
|
141
|
+
# set everything
|
142
|
+
AllGems.defaulterize
|
143
|
+
AllGems.data_directory = datadir
|
144
|
+
AllGems.doc_format = formatter unless formatter.nil?
|
145
|
+
AllGems.logger = Logger.new(logto, level)
|
146
|
+
AllGems.allgems = all
|
147
|
+
|
148
|
+
# are we just giving out configuration?
|
149
|
+
if(config_output)
|
150
|
+
puts <<-EOC
|
151
|
+
Apache virtual host configuration:
|
152
|
+
|
153
|
+
<VirtualHost *:80>
|
154
|
+
# Uncomment the next line and add your servers name if wanted
|
155
|
+
# ServerName YourServerNameHere.com
|
156
|
+
|
157
|
+
SetEnv DATA_DIR #{AllGems.data_directory}
|
158
|
+
SetEnv DATA_DB #{dbfile}
|
159
|
+
|
160
|
+
Alias /docs #{AllGems.data_directory}
|
161
|
+
Alias /docs/ #{AllGems.data_directory}/
|
162
|
+
Alias #{AllGems.data_directory}/ #{AllGems.data_directory}/
|
163
|
+
<Location "/docs">
|
164
|
+
PassengerEnabled off
|
165
|
+
RewriteEngine on
|
166
|
+
RewriteRule allgems.db / [R]
|
167
|
+
RewriteRule ^.+?rdoc.*?/tmp/allgems/(.*)$ /docs/$1 [R]
|
168
|
+
</Location>
|
169
|
+
<Location "#{AllGems.data_directory}">
|
170
|
+
PassengerEnabled off
|
171
|
+
RewriteEngine on
|
172
|
+
RewriteRule .*?rdocs/.*?/tmp/allgems/(.+)$ /docs/$1 [R]
|
173
|
+
RewriteRule (.*) /docs/$1 [R]
|
174
|
+
</Location>
|
175
|
+
<Directory #{AllGems.data_directory}>
|
176
|
+
Options -Indexes
|
177
|
+
</Directory>
|
178
|
+
ErrorDocument 403 /gems
|
179
|
+
DocumentRoot #{AllGems.public_directory}
|
180
|
+
</VirtualHost>
|
181
|
+
|
182
|
+
Required apache modules:
|
183
|
+
mod_rewrite
|
184
|
+
mod_env
|
185
|
+
mod_alias
|
186
|
+
EOC
|
187
|
+
exit SUCCESS
|
188
|
+
end
|
189
|
+
|
190
|
+
# are we supposed to be a daemon?
|
191
|
+
if(daemonize)
|
192
|
+
if(RUBY_VERSION > '1.9.0')
|
193
|
+
Process.daemon
|
194
|
+
else
|
195
|
+
if(pid = fork)
|
196
|
+
Signal.trap('HUP', 'IGNORE')
|
197
|
+
Process.detach(pid)
|
198
|
+
exit
|
199
|
+
end
|
200
|
+
end
|
201
|
+
end
|
202
|
+
|
203
|
+
# Make sure we play nice before we get started
|
204
|
+
|
205
|
+
Process.setpriority(Process::PRIO_PROCESS, 0, 19) unless ON_WINDOWS
|
206
|
+
|
207
|
+
# finally, lets get going
|
208
|
+
|
209
|
+
require 'allgems/Runner'
|
210
|
+
runner = AllGems::Runner.new(:db_path => dbfile, :runners => runners, :interval => interval)
|
211
|
+
trap('SIGINT'){runner.stop(true)}
|
212
|
+
trap('SIGHUP'){runner.stop; runner.do_sync} unless ON_WINDOWS
|
213
|
+
runner.do_sync
|
214
|
+
runner.stop unless daemonize
|
215
|
+
exit SUCCESS
|
data/config.ru
ADDED
@@ -0,0 +1,11 @@
|
|
1
|
+
%w(allgems allgems/App haml sass sequel sequel/extensions/pagination).each{|f|require f}
|
2
|
+
|
3
|
+
AllGems.defaulterize
|
4
|
+
AllGems.data_directory = ENV['DATA_DIR']
|
5
|
+
AllGems.initialize_db(Sequel.connect("sqlite://#{ENV['DATA_DB']}"))
|
6
|
+
|
7
|
+
disable :run
|
8
|
+
AllGems::App.set({
|
9
|
+
:environment => :development
|
10
|
+
})
|
11
|
+
run AllGems::App
|
data/lib/allgems.rb
ADDED
@@ -0,0 +1,69 @@
|
|
1
|
+
require 'logger'
|
2
|
+
module AllGems
|
3
|
+
class << self
|
4
|
+
attr_accessor :data_directory, :logger, :db, :allgems
|
5
|
+
def defaulterize
|
6
|
+
@data_directory = nil
|
7
|
+
@doc_format = ['rdoc']
|
8
|
+
@logger = Logger.new(nil)
|
9
|
+
@db = nil
|
10
|
+
@tg = nil
|
11
|
+
@ti = nil
|
12
|
+
@allgems = true
|
13
|
+
@refresh = Time.now.to_i + 3600
|
14
|
+
end
|
15
|
+
# db: Sequel::Database
|
16
|
+
# Run any migrations needed
|
17
|
+
# NOTE: Call before using the database
|
18
|
+
def initialize_db(db)
|
19
|
+
self.db = db
|
20
|
+
require 'sequel/extensions/migration'
|
21
|
+
Sequel::Migrator.apply(db, "#{File.expand_path(__FILE__).gsub(/\/[^\/]+$/, '')}/allgems/migrations")
|
22
|
+
end
|
23
|
+
# Format for documentation
|
24
|
+
def doc_format
|
25
|
+
@doc_format
|
26
|
+
end
|
27
|
+
# f:: format
|
28
|
+
# Sets format for documentation. Should be one of: hanna, rdoc, sdoc
|
29
|
+
def doc_format=(f)
|
30
|
+
@doc_format = []
|
31
|
+
f.split(',').each do |type|
|
32
|
+
type = type.to_sym
|
33
|
+
raise ArgumentError.new("Valid types: hanna, sdoc, rdoc") unless [:hanna,:sdoc,:rdoc].include?(type)
|
34
|
+
@doc_format << type
|
35
|
+
end
|
36
|
+
end
|
37
|
+
# Location of public directory
|
38
|
+
def public_directory
|
39
|
+
File.expand_path("#{__FILE__}/../../public")
|
40
|
+
end
|
41
|
+
# Total number of unique gem names installed
|
42
|
+
def total_gems
|
43
|
+
return 0 if self.db.nil?
|
44
|
+
if(@tg.nil? || Time.now.to_i > @refresh)
|
45
|
+
@tg = self.db[:gems].count
|
46
|
+
@refresh = Time.now.to_i + 3600
|
47
|
+
end
|
48
|
+
@tg
|
49
|
+
end
|
50
|
+
# Total number of actual gems installed
|
51
|
+
def total_installs
|
52
|
+
return 0 if self.db.nil?
|
53
|
+
if(@ti.nil? || Time.now.to_i > @refresh)
|
54
|
+
@ti = self.db[:versions].count
|
55
|
+
@refresh = Time.now.to_i + 3600
|
56
|
+
end
|
57
|
+
@ti
|
58
|
+
end
|
59
|
+
# Newest gem based on release date
|
60
|
+
def newest_gem
|
61
|
+
g = AllGems.db[:versions].join(:gems, :id => :gem_id).order(:release.desc).limit(1).select(:name, :release).first
|
62
|
+
{:name => g[:name], :release => g[:release]}
|
63
|
+
end
|
64
|
+
# Path to hanna hack
|
65
|
+
def hanna_hack
|
66
|
+
File.expand_path("#{__FILE__}/../allgems/hanna_hack.rb")
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
data/lib/allgems/App.rb
ADDED
@@ -0,0 +1,92 @@
|
|
1
|
+
require 'sinatra'
|
2
|
+
require 'allgems/ViewHelpers'
|
3
|
+
require 'allgems/Specer'
|
4
|
+
require 'rubygems/specification'
|
5
|
+
|
6
|
+
module AllGems
|
7
|
+
|
8
|
+
class App < ::Sinatra::Default
|
9
|
+
|
10
|
+
include AllGems::ViewHelpers
|
11
|
+
|
12
|
+
@@root = File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))
|
13
|
+
set :root, @@root
|
14
|
+
set :app_file, __FILE__
|
15
|
+
|
16
|
+
before do
|
17
|
+
@environment = options.environment
|
18
|
+
end
|
19
|
+
|
20
|
+
get '/stylesheets/:stylesheet.css' do
|
21
|
+
sass params[:stylesheet].to_sym
|
22
|
+
end
|
23
|
+
|
24
|
+
# root is nothing, so redirect people on their way
|
25
|
+
get '/' do
|
26
|
+
redirect '/gems'
|
27
|
+
end
|
28
|
+
|
29
|
+
# generate the gem listing
|
30
|
+
get '/gems/?' do
|
31
|
+
show_layout = params[:layout] != 'false'
|
32
|
+
@show_as = params[:as] && params[:as] == 'table' ? 'table' : 'columns'
|
33
|
+
set = AllGems.db[:gems].order(:name.asc)
|
34
|
+
@page = params[:page] ? params[:page].to_i : 1
|
35
|
+
if(@search = params[:search])
|
36
|
+
set = do_search(params[:search])
|
37
|
+
if(set.count == 1)
|
38
|
+
redirect "/gems/#{set.first[:name]}" # send user on their way if we only get one result
|
39
|
+
end
|
40
|
+
end
|
41
|
+
@gems = set.paginate(@page, 30)
|
42
|
+
haml "gems_#{@show_as}".to_sym, :layout => show_layout
|
43
|
+
end
|
44
|
+
|
45
|
+
# send the the correct place
|
46
|
+
get %r{/gems/([\w\-\_]+)/?([\d\.]+)?/?} do
|
47
|
+
gem = params[:captures][0]
|
48
|
+
version = params[:captures].size > 1 ? params[:captures][1] : nil
|
49
|
+
@gem = load_gem_spec(gem, version)
|
50
|
+
@versions = AllGems.db[:versions].join(:gems, :id => :gem_id).filter(:name => gem).order(:version.desc).select(:version, :release)
|
51
|
+
haml :gem
|
52
|
+
end
|
53
|
+
|
54
|
+
get %r{/doc/(.+)} do
|
55
|
+
parts = params[:captures][0].split('/')
|
56
|
+
@gem_name = parts[0]
|
57
|
+
@gem_version = parts[1]
|
58
|
+
@path = "/docs/#{params[:captures][0]}"
|
59
|
+
haml :doc, :layout => false
|
60
|
+
end
|
61
|
+
|
62
|
+
get %r{/load/([^/]+)/(.+)/?} do
|
63
|
+
@gem_name = params[:captures][0]
|
64
|
+
@gem_version = params[:captures][1]
|
65
|
+
haml :load, :layout => false
|
66
|
+
end
|
67
|
+
|
68
|
+
private
|
69
|
+
|
70
|
+
def load_gem_spec(gem, version=nil)
|
71
|
+
version ||= get_latest(gem)
|
72
|
+
raise 'failed gem' unless version
|
73
|
+
Specer.get_spec(gem, version)
|
74
|
+
end
|
75
|
+
|
76
|
+
# gem:: name of the gem
|
77
|
+
# Returns the latest version of the given gem or nil
|
78
|
+
def get_latest(gem)
|
79
|
+
AllGems.db[:versions].join(:gems, :id => :gem_id).filter(:name => gem).order(:version.desc).limit(1).select(:version).map(:version)[0]
|
80
|
+
end
|
81
|
+
|
82
|
+
# terms:: terms to search on
|
83
|
+
#
|
84
|
+
def do_search(terms)
|
85
|
+
terms = terms.split.map{|t|"%#{t}%"}
|
86
|
+
names = AllGems.db[:gems].filter("#{[].fill('name LIKE ?', 0, terms.size).join(' OR ')}", *terms).order(:name.asc)
|
87
|
+
desc = AllGems.db[:gems].filter("#{[].fill('description LIKE ?', 0, terms.size).join(' OR ')}", *terms).order(:name.asc)
|
88
|
+
summ = AllGems.db[:gems].filter("#{[].fill('summary LIKE ?', 0, terms.size).join(' OR ')}", *terms).order(:name.asc)
|
89
|
+
names.union(desc).union(summ)
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
File without changes
|
@@ -0,0 +1,229 @@
|
|
1
|
+
require 'net/http'
|
2
|
+
require 'uri'
|
3
|
+
require 'rubygems/installer'
|
4
|
+
require 'allgems/Indexer'
|
5
|
+
|
6
|
+
module AllGems
|
7
|
+
class GemWorker
|
8
|
+
class << self
|
9
|
+
attr_reader :pool
|
10
|
+
# Get the worker ready to go
|
11
|
+
def setup
|
12
|
+
@glock = Mutex.new
|
13
|
+
@slock = Mutex.new
|
14
|
+
@pool = ActionPool::Pool.new(:max_threads => 10, :a_to => 60*5)
|
15
|
+
end
|
16
|
+
# :name:: name of the gem
|
17
|
+
# :version:: version of the gem
|
18
|
+
# :database:: database connection
|
19
|
+
def process(args)
|
20
|
+
AllGems.logger.info "Processing gem: #{args[:name]}-#{args[:version]}"
|
21
|
+
spec,uri = self.get_spec(args[:name], args[:version])
|
22
|
+
raise NameError.new("Name not found: #{args[:name]} - #{args[:version]}") if spec.nil?
|
23
|
+
basedir = "#{AllGems.data_directory}/#{spec.name}/#{spec.version.version}"
|
24
|
+
FileUtils.mkdir_p "#{basedir}/unpack"
|
25
|
+
AllGems.logger.info "Created new directory: #{basedir}/unpack"
|
26
|
+
gempath = self.fetch(spec, uri, "#{basedir}/#{spec.full_name}.gem")
|
27
|
+
self.unpack(gempath, basedir)
|
28
|
+
self.generate_documentation(spec, basedir)
|
29
|
+
self.save_data(spec, args[:database])
|
30
|
+
end
|
31
|
+
|
32
|
+
# name:: name of gem
|
33
|
+
# version:: version of gem
|
34
|
+
# Fetches the Gem::Specification for the given gem
|
35
|
+
def get_spec(name, version)
|
36
|
+
AllGems.logger.info "Fetching gemspec for #{name}-#{version}"
|
37
|
+
dep = Gem::Dependency.new(name, version)
|
38
|
+
spec = nil
|
39
|
+
@glock.synchronize{spec = Gem::SpecFetcher.fetcher.fetch dep, true}
|
40
|
+
spec[0]
|
41
|
+
end
|
42
|
+
|
43
|
+
# spec:: Gem::Specification
|
44
|
+
# uri:: URI of gem files home
|
45
|
+
# save_path:: path to save gem
|
46
|
+
# Fetch the gem file from the server. Returns the path to the gem
|
47
|
+
# on the local machine
|
48
|
+
def fetch(spec, uri, save_path)
|
49
|
+
AllGems.logger.info "Fetching gem from: #{uri}/gems/#{spec.full_name}.gem"
|
50
|
+
FileUtils.touch(save_path)
|
51
|
+
begin
|
52
|
+
remote_path = "#{uri}/gems/#{spec.full_name}.gem"
|
53
|
+
remote_uri = URI.parse(remote_path)
|
54
|
+
file = File.open(save_path, 'wb')
|
55
|
+
file.write(self.fetch_remote(remote_uri))
|
56
|
+
file.close
|
57
|
+
save_path
|
58
|
+
rescue StandardError => boom
|
59
|
+
raise FetchError.new(spec.name, spec.version, remote_path, boom)
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
# uri:: URI of gem
|
64
|
+
# depth:: number of times called
|
65
|
+
# Fetch gem from given URI
|
66
|
+
def fetch_remote(uri, depth=0)
|
67
|
+
raise IOError.new("Depth too deep") if depth > 9
|
68
|
+
response = Net::HTTP.get_response(uri)
|
69
|
+
if(response.is_a?(Net::HTTPSuccess))
|
70
|
+
response.body
|
71
|
+
elsif(response.is_a?(Net::HTTPRedirection))
|
72
|
+
self.fetch_remote(URI.parse(response['location']), depth + 1)
|
73
|
+
else
|
74
|
+
raise IOError.new("Unknown response type: #{response}")
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
# gempath:: path to gem file
|
79
|
+
# newname:: path to move gem file to
|
80
|
+
# Moves the gem to the given location
|
81
|
+
def save(gempath, newpath)
|
82
|
+
AllGems.logger.info "Moving #{gempath} to #{newpath}"
|
83
|
+
FileUtils.mv gempath, newpath
|
84
|
+
newpath
|
85
|
+
end
|
86
|
+
|
87
|
+
# path:: path to the gem file
|
88
|
+
# basedir:: directory to unpack in
|
89
|
+
# depth:: number of times called
|
90
|
+
# Unpacks the gem into the basedir under the 'unpack' directory
|
91
|
+
def unpack(path, basedir, depth=0)
|
92
|
+
AllGems.logger.info "Unpacking gem: #{path}"
|
93
|
+
begin
|
94
|
+
Gem::Installer.new(path, :unpack => true).unpack "#{basedir}/unpack"
|
95
|
+
FileUtils.chmod_R(0755, "#{basedir}/unpack") # fix any bad permissions
|
96
|
+
FileUtils.rm(path)
|
97
|
+
rescue
|
98
|
+
if(File.size(path) < 1 || depth > 10)
|
99
|
+
raise IOError.new("Failed to unpack gem: #{path}") unless self.direct_unpack(path, basedir)
|
100
|
+
else
|
101
|
+
self.unpack(path, basedir, depth+1)
|
102
|
+
end
|
103
|
+
end
|
104
|
+
true
|
105
|
+
end
|
106
|
+
|
107
|
+
# path:: path to the gem file
|
108
|
+
# basedir:: directory to unpack in
|
109
|
+
# Last ditch effort to unpack the gem
|
110
|
+
def direct_unpack(path, basedir)
|
111
|
+
AllGems.logger.warn "Attempting forcible unpack on: #{path}"
|
112
|
+
unpackdir = path.slice(0, path.rindex('.'))
|
113
|
+
self.run_command("cd #{basedir} && gem unpack #{path} && mv #{unpackdir}/* #{basedir}/unpack/ && rm -rf #{unpackdir}")
|
114
|
+
end
|
115
|
+
|
116
|
+
# dir:: base directory location of gem contents
|
117
|
+
# Generates the documentation of the given directory of ruby code. Appends
|
118
|
+
# 'unpack' to the given directory for code discovery. Documentation will
|
119
|
+
# be output in "#{dir}/doc"
|
120
|
+
def generate_documentation(spec, dir)
|
121
|
+
AllGems.logger.info "Generating documentation for #{spec.full_name}"
|
122
|
+
AllGems.doc_format.each do |f|
|
123
|
+
command = nil
|
124
|
+
args = []
|
125
|
+
case f.to_sym
|
126
|
+
when :rdoc
|
127
|
+
command = 'rdoc'
|
128
|
+
args << '--format=darkfish' << '-aFNqH' << "--op=#{dir}/doc/rdoc" << "#{dir}/unpack"
|
129
|
+
when :sdoc
|
130
|
+
command = 'sdoc'
|
131
|
+
args << '-T direct' << "-o #{dir}/doc/sdoc" << "#{dir}/unpack"
|
132
|
+
when :hanna
|
133
|
+
command = "ruby #{AllGems.hanna_hack}"
|
134
|
+
args << "-o #{dir}/doc/hanna" << "#{dir}/unpack"
|
135
|
+
else
|
136
|
+
next # if we don't know what to do with it, skip it
|
137
|
+
end
|
138
|
+
action = lambda do |spec, dir, command, args, f|
|
139
|
+
FileUtils.rm_r("#{dir}/doc/#{f}", :force => true) # make sure we are clean before we get dirty
|
140
|
+
result = self.run_command("#{command} #{args}")
|
141
|
+
raise DocError.new(spec.name, spec.version) unless result
|
142
|
+
AllGems.logger.info "Completed documentation for #{spec.full_name}"
|
143
|
+
FileUtils.chmod_R(0755, "#{dir}/doc/#{f}") # fix any bad permissions
|
144
|
+
Indexer.index_gem(spec, "#{dir}/doc/#{f}", f)
|
145
|
+
end
|
146
|
+
@pool << [action, [spec, dir.dup, command.dup, args.join(' '), f]]
|
147
|
+
end
|
148
|
+
end
|
149
|
+
|
150
|
+
# command:: command to run
|
151
|
+
# return_output:: return output
|
152
|
+
# Runs a command. Returns true if status returns 0. If return_output is true,
|
153
|
+
# return value is: [status, output]
|
154
|
+
def run_command(command, return_output=false)
|
155
|
+
AllGems.logger.debug "Command to be executed: #{command}"
|
156
|
+
pro = nil
|
157
|
+
output = []
|
158
|
+
status = nil
|
159
|
+
begin
|
160
|
+
pro = IO.popen(command)
|
161
|
+
Process.setpriority(Process::PRIO_PROCESS, pro.pid, 19) unless ON_WINDOWS
|
162
|
+
until(pro.closed? || pro.eof?)
|
163
|
+
output << pro.gets
|
164
|
+
end
|
165
|
+
ensure
|
166
|
+
unless(pro.nil?)
|
167
|
+
pid, status = Process.waitpid2(pro.pid)
|
168
|
+
else
|
169
|
+
status = 1
|
170
|
+
end
|
171
|
+
end
|
172
|
+
return return_output ? [status == 0, output.join] : status == 0
|
173
|
+
end
|
174
|
+
|
175
|
+
# spec:: Gem::Specification
|
176
|
+
# Save data to the database about this gem
|
177
|
+
def save_data(spec, db)
|
178
|
+
AllGems.logger.info "Saving meta data for #{spec.full_name}"
|
179
|
+
@slock.synchronize do
|
180
|
+
gid = db[:gems].filter(:name => spec.name).first
|
181
|
+
gid = gid.nil? ? db[:gems].insert(:name => spec.name) : gid[:id]
|
182
|
+
pid = db[:platforms].filter(:platform => spec.platform).first
|
183
|
+
pid = pid.nil? ? db[:platforms].insert(:platform => spec.platform) : pid[:id]
|
184
|
+
vid = db[:versions] << {:version => spec.version.version, :gem_id => gid, :platform_id => pid, :release => spec.date}
|
185
|
+
db[:specs] << {:version_id => vid, :spec => [Marshal.dump(spec)].pack('m')}
|
186
|
+
db[:gems].filter(:id => gid).update(:summary => spec.summary)
|
187
|
+
db[:gems].filter(:id => gid).update(:description => spec.description)
|
188
|
+
end
|
189
|
+
AllGems.logger.info "Meta data saving complete for #{spec.full_name}"
|
190
|
+
true
|
191
|
+
end
|
192
|
+
|
193
|
+
# Make a parent error class that we can specialize
|
194
|
+
class Error < StandardError
|
195
|
+
attr_reader :original
|
196
|
+
def initialize(e=nil)
|
197
|
+
@original = e.nil? ? self : e
|
198
|
+
end
|
199
|
+
end
|
200
|
+
|
201
|
+
# Exception class for failed documentation creation
|
202
|
+
class DocError < Error
|
203
|
+
attr_reader :gem_name, :gem_version
|
204
|
+
def initialize(gn, gv, e=nil)
|
205
|
+
super(e)
|
206
|
+
@gem_name = gn
|
207
|
+
@gem_version = gv
|
208
|
+
end
|
209
|
+
def to_s
|
210
|
+
"Failed to create documentation for: #{@gem_name}-#{@gem_version}."
|
211
|
+
end
|
212
|
+
end
|
213
|
+
|
214
|
+
# Exception class for failed gem fetching
|
215
|
+
class FetchError < Error
|
216
|
+
attr_reader :gem_name, :gem_version, :uri
|
217
|
+
def initialize(gn, gv, u, e=nil)
|
218
|
+
super(e)
|
219
|
+
@gem_name = gn
|
220
|
+
@gem_version = gv
|
221
|
+
@uri = u
|
222
|
+
end
|
223
|
+
def to_s
|
224
|
+
"Failed to fetch #{@gem_name}-#{@gem_version}.gem from #{uri}"
|
225
|
+
end
|
226
|
+
end
|
227
|
+
end
|
228
|
+
end
|
229
|
+
end
|