allgems 0.0.1

data/README.rdoc

@@ -0,0 +1,16 @@
+= AllGems
+
+== Description
+
+AllGems is a simple project with a hefty goal: Provide easy access to documentation for all RubyGem files.
+
+== How?
+
+==== Backend
+
+AllGems runs a small daemon that monitors the index of any sources registered with the locally installed RubyGems. This daemon automatically retrieves, unpacks, and generates documentation on libraries as they are released.
+
+==== Frontend
+
+AllGems uses a heavily modified version of Gembox to allow access to available documentation.
+

data/allgems.gemspec

@@ -0,0 +1,24 @@
+spec = Gem::Specification.new do |s|
+    s.name              = 'allgems'
+    s.author            = %q(spox)
+    s.email             = %q(spox@modspox.com)
+    s.version           = '0.0.1'
+    s.summary           = %q(Tools to document the world)
+    s.platform          = Gem::Platform::RUBY
+    s.files             = Dir['**/*']
+    s.rdoc_options      = %w(--title AllGems --main README.rdoc --line-numbers)
+    s.extra_rdoc_files  = %w(README.rdoc CHANGELOG)
+    s.require_paths     = %w(lib)
+    s.executables       = %w(allgems)
+    s.required_ruby_version = '>= 1.8.6'
+    s.homepage          = %q(http://github.com/spox/allgems)
+    s.description       = 'AllGems is a tool to provide comprehensive gem documentation for an entire index'
+    s.add_dependency 'sequel'
+    s.add_dependency 'ActionPool'
+    s.add_dependency 'ActionTimer'
+    s.add_dependency 'haml'
+    s.add_dependency 'rdoc'
+    s.add_dependency 'sdoc'
+    s.add_dependency 'hanna'
+    s.add_dependency 'nokogiri'
+end

data/bin/allgems

@@ -0,0 +1,215 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+SUCCESS = 0
+CONFIGURATION_ERROR = 1
+
+def display_help
+    $stdout.puts "\t\tAllGems - Document Everything
+usage: allgems [opts]
+--config -c:\t\t\t\tOutput apache configuration
+--notall -n:\t\t\t\tDon't fetch all versions
+--daemonize -D:\t\t\t\tRun as a daemon
+--datadir -d /path:\t\t\tData directory for documents/gems (defaults to /tmp/allgems)
+--dbfile -b /path/file.db\t\tPath to sqlite file (defaults datadir/allgems.db)
+--formatter -f rdoc[,hanna,sdoc]:\t\tRDoc formatter (defaults to rdoc (which is darkfish))
+--interval -i \\d+:\t\t\tSeconds to wait before updating documents (defaults to 1 hour)
+--runners -r \\d+:\t\t\tNumber of threads to use (defaults to 10)
+--log -L [/path/file]:\t\t\tTurn logging on (no argument defaults to STDOUT)
+--verbosity -V [(debug|info|warn|fatal)]:\tLogging level (no argument defaults to info)
+--help -h:\t\t\t\tDisplay this help text"
+end
+
+# load in our libraries
+['logger', 'fileutils', 'getoptlong', 'allgems', 'sequel'].each{|f|require f}
+
+# figure out what system we are on
+require 'rbconfig'
+ON_WINDOWS = Config::CONFIG['host_os'] =~ /mswin|mingw/
+
+# set default values
+config_output = false
+daemonize = false
+datadir = '/tmp/allgems'
+formatter = nil
+interval = 3600 # default to hourly checks
+runners = 10 # default to 10 threads
+dbfile = nil
+logto = nil
+level = Logger::INFO
+all = true
+
+# figure out what we are doing
+opts = GetoptLong.new(
+    ['--daemonize', '-D', GetoptLong::NO_ARGUMENT],
+    ['--datadir', '-d', GetoptLong::REQUIRED_ARGUMENT],
+    ['--dbfile', '-b', GetoptLong::REQUIRED_ARGUMENT],
+    ['--formatter', '-f', GetoptLong::REQUIRED_ARGUMENT],
+    ['--interval', '-i', GetoptLong::REQUIRED_ARGUMENT],
+    ['--runners', '-r', GetoptLong::REQUIRED_ARGUMENT],
+    ['--version', '-v', GetoptLong::NO_ARGUMENT],
+    ['--log', '-L', GetoptLong::OPTIONAL_ARGUMENT],
+    ['--verbosity', '-V', GetoptLong::OPTIONAL_ARGUMENT],
+    ['--help', '-h', GetoptLong::NO_ARGUMENT],
+    ['--config', '-c', GetoptLong::NO_ARGUMENT],
+    ['--notall', '-n', GetoptLong::NO_ARGUMENT]
+)
+
+opts.each do |opt, arg|
+    case opt
+        when '--daemonize'
+            daemonize = true
+        when '--datadir'
+            datadir = arg
+        when '--formatter'
+            formatter = arg
+        when '--interval'
+            interval = arg.to_i
+        when '--runners'
+            runners = arg.to_i
+        when '--dbfile'
+            dbfile = arg
+        when '--version'
+            puts "VERSION INFO"
+        when '--log'
+            logto = arg.empty? ? $stdout: arg
+        when '--verbosity'
+            case arg
+                when 'debug'
+                    level = Logger::DEBUG
+                when 'info'
+                    level = Logger::INFO
+                when 'warn'
+                    level = Logger::WARN
+                when 'error'
+                    level = Logger::ERROR
+                when 'fatal'
+                    level = Logger::FATAL
+                else
+                    level = Logger::INFO
+            end
+        when '--notall'
+            all = false
+        when '--config'
+            config_output = true
+        when '--help'
+            display_help
+            exit SUCCESS
+    end
+end
+
+dbfile = "#{datadir}/allgems.db" if dbfile.nil?
+interval = nil unless daemonize
+
+# yell at user for bad values
+if(interval && interval <= 0)
+    $stderr.puts "ERROR: interval value must be greater than 1"
+    exit CONFIGURATION_ERROR
+end
+if(runners <= 0)
+    $stderr.puts "ERROR: runners value must be greater than 1"
+    exit CONFIGURATION_ERROR
+end
+unless(File.writable?(datadir))
+    created = false
+    #check if we can/should create it
+    unless(File.exists?(datadir))
+        begin
+            FileUtils.mkdir_p datadir
+            created = true
+        rescue SystemCallError
+            created = false
+        end
+    end
+    datadir = File.expand_path(datadir)
+    unless(created)
+        $stderr.puts "ERROR: data directory is not writable. (#{datadir})"
+        exit CONFIGURATION_ERROR
+    end
+end
+unless(File.exists?(dbfile))
+    begin
+        FileUtils.touch dbfile
+    rescue SystemCallError
+    end
+end
+unless(File.writable?(dbfile))
+    $stderr.puts "ERROR: database file is not writable. (#{dbfile})"
+    exit CONFIGURATION_ERROR
+end
+
+# set everything
+AllGems.defaulterize
+AllGems.data_directory = datadir
+AllGems.doc_format = formatter unless formatter.nil?
+AllGems.logger = Logger.new(logto, level)
+AllGems.allgems = all
+
+# are we just giving out configuration?
+if(config_output)
+puts <<-EOC
+Apache virtual host configuration:
+
+<VirtualHost *:80>
+        # Uncomment the next line and add your servers name if wanted
+        # ServerName YourServerNameHere.com
+
+        SetEnv DATA_DIR #{AllGems.data_directory}
+        SetEnv DATA_DB #{dbfile}
+        
+        Alias /docs #{AllGems.data_directory}
+        Alias /docs/ #{AllGems.data_directory}/
+        Alias #{AllGems.data_directory}/ #{AllGems.data_directory}/
+        <Location "/docs">
+                PassengerEnabled off
+                RewriteEngine on
+                RewriteRule allgems.db / [R]
+                RewriteRule ^.+?rdoc.*?/tmp/allgems/(.*)$ /docs/$1 [R]
+        </Location>
+        <Location "#{AllGems.data_directory}">
+                PassengerEnabled off
+                RewriteEngine on
+                RewriteRule .*?rdocs/.*?/tmp/allgems/(.+)$ /docs/$1 [R]
+                RewriteRule (.*) /docs/$1 [R]
+        </Location>
+        <Directory #{AllGems.data_directory}>
+                Options -Indexes
+        </Directory>
+        ErrorDocument 403 /gems
+        DocumentRoot #{AllGems.public_directory}
+</VirtualHost>
+
+Required apache modules:
+mod_rewrite
+mod_env
+mod_alias
+EOC
+exit SUCCESS
+end
+
+# are we supposed to be a daemon?
+if(daemonize)
+    if(RUBY_VERSION > '1.9.0')
+        Process.daemon
+    else
+        if(pid = fork)
+            Signal.trap('HUP', 'IGNORE')
+            Process.detach(pid)
+            exit
+        end
+    end
+end
+
+# Make sure we play nice before we get started
+
+Process.setpriority(Process::PRIO_PROCESS, 0, 19) unless ON_WINDOWS
+
+# finally, lets get going
+
+require 'allgems/Runner'
+runner = AllGems::Runner.new(:db_path => dbfile, :runners => runners, :interval => interval)
+trap('SIGINT'){runner.stop(true)}
+trap('SIGHUP'){runner.stop; runner.do_sync} unless ON_WINDOWS
+runner.do_sync
+runner.stop unless daemonize
+exit SUCCESS

data/config.ru

@@ -0,0 +1,11 @@
+%w(allgems allgems/App haml sass sequel sequel/extensions/pagination).each{|f|require f}
+
+AllGems.defaulterize
+AllGems.data_directory = ENV['DATA_DIR']
+AllGems.initialize_db(Sequel.connect("sqlite://#{ENV['DATA_DB']}"))
+
+disable :run
+AllGems::App.set({
+  :environment => :development
+})
+run AllGems::App

data/lib/allgems.rb

@@ -0,0 +1,69 @@
+require 'logger'
+module AllGems
+    class << self
+        attr_accessor :data_directory, :logger, :db, :allgems
+        def defaulterize
+            @data_directory = nil
+            @doc_format = ['rdoc']
+            @logger = Logger.new(nil)
+            @db = nil
+            @tg = nil
+            @ti = nil
+            @allgems = true
+            @refresh = Time.now.to_i + 3600
+        end
+        # db: Sequel::Database
+        # Run any migrations needed
+        # NOTE: Call before using the database
+        def initialize_db(db)
+            self.db = db
+            require 'sequel/extensions/migration'
+            Sequel::Migrator.apply(db, "#{File.expand_path(__FILE__).gsub(/\/[^\/]+$/, '')}/allgems/migrations")
+        end
+        # Format for documentation
+        def doc_format
+            @doc_format
+        end
+        # f:: format
+        # Sets format for documentation. Should be one of: hanna, rdoc, sdoc
+        def doc_format=(f)
+            @doc_format = []
+            f.split(',').each do |type|
+                type = type.to_sym
+                raise ArgumentError.new("Valid types: hanna, sdoc, rdoc") unless [:hanna,:sdoc,:rdoc].include?(type)
+                @doc_format << type
+            end
+        end
+        # Location of public directory
+        def public_directory
+            File.expand_path("#{__FILE__}/../../public")
+        end
+        # Total number of unique gem names installed
+        def total_gems
+            return 0 if self.db.nil?
+            if(@tg.nil? || Time.now.to_i > @refresh)
+                @tg = self.db[:gems].count
+                @refresh = Time.now.to_i + 3600
+            end
+            @tg
+        end
+        # Total number of actual gems installed
+        def total_installs
+            return 0 if self.db.nil?
+            if(@ti.nil? || Time.now.to_i > @refresh)
+                @ti = self.db[:versions].count
+                @refresh = Time.now.to_i + 3600
+            end
+            @ti
+        end
+        # Newest gem based on release date
+        def newest_gem
+            g = AllGems.db[:versions].join(:gems, :id => :gem_id).order(:release.desc).limit(1).select(:name, :release).first
+            {:name => g[:name], :release => g[:release]}
+        end
+        # Path to hanna hack
+        def hanna_hack
+            File.expand_path("#{__FILE__}/../allgems/hanna_hack.rb")
+        end
+    end
+end

data/lib/allgems/App.rb

@@ -0,0 +1,92 @@
+require 'sinatra'
+require 'allgems/ViewHelpers'
+require 'allgems/Specer'
+require 'rubygems/specification'
+
+module AllGems
+    
+    class App < ::Sinatra::Default
+
+        include AllGems::ViewHelpers
+
+        @@root = File.expand_path(File.join(File.dirname(__FILE__), '..', '..'))
+        set :root, @@root
+        set :app_file, __FILE__
+
+        before do
+            @environment = options.environment
+        end
+        
+        get '/stylesheets/:stylesheet.css' do
+            sass params[:stylesheet].to_sym
+        end
+        
+        # root is nothing, so redirect people on their way
+        get '/' do
+            redirect '/gems'
+        end
+
+        # generate the gem listing
+        get '/gems/?' do
+            show_layout = params[:layout] != 'false'
+            @show_as = params[:as] && params[:as] == 'table' ? 'table' : 'columns'
+            set = AllGems.db[:gems].order(:name.asc)
+            @page = params[:page] ? params[:page].to_i : 1
+            if(@search = params[:search])
+                set = do_search(params[:search])
+                if(set.count == 1)
+                    redirect "/gems/#{set.first[:name]}" # send user on their way if we only get one result
+                end
+            end
+            @gems = set.paginate(@page, 30)
+            haml "gems_#{@show_as}".to_sym, :layout => show_layout
+        end
+
+        # send the the correct place
+        get %r{/gems/([\w\-\_]+)/?([\d\.]+)?/?} do
+            gem = params[:captures][0]
+            version = params[:captures].size > 1 ? params[:captures][1] : nil
+            @gem = load_gem_spec(gem, version)
+            @versions = AllGems.db[:versions].join(:gems, :id => :gem_id).filter(:name => gem).order(:version.desc).select(:version, :release)
+            haml :gem
+        end
+
+        get %r{/doc/(.+)} do
+            parts = params[:captures][0].split('/')
+            @gem_name = parts[0]
+            @gem_version = parts[1]
+            @path = "/docs/#{params[:captures][0]}"
+            haml :doc, :layout => false
+        end
+
+        get %r{/load/([^/]+)/(.+)/?} do
+            @gem_name = params[:captures][0]
+            @gem_version = params[:captures][1]
+            haml :load, :layout => false
+        end
+
+        private
+
+        def load_gem_spec(gem, version=nil)
+            version ||= get_latest(gem)
+            raise 'failed gem' unless version
+            Specer.get_spec(gem, version)
+        end
+
+        # gem:: name of the gem
+        # Returns the latest version of the given gem or nil
+        def get_latest(gem)
+            AllGems.db[:versions].join(:gems, :id => :gem_id).filter(:name => gem).order(:version.desc).limit(1).select(:version).map(:version)[0]
+        end
+
+        # terms:: terms to search on
+        # 
+        def do_search(terms)
+            terms = terms.split.map{|t|"%#{t}%"}
+            names = AllGems.db[:gems].filter("#{[].fill('name LIKE ?', 0, terms.size).join(' OR ')}", *terms).order(:name.asc)
+            desc = AllGems.db[:gems].filter("#{[].fill('description LIKE ?', 0, terms.size).join(' OR ')}", *terms).order(:name.asc)
+            summ = AllGems.db[:gems].filter("#{[].fill('summary LIKE ?', 0, terms.size).join(' OR ')}", *terms).order(:name.asc)
+            names.union(desc).union(summ)
+        end
+    end
+end

data/lib/allgems/GemWorker.rb

@@ -0,0 +1,229 @@
+require 'net/http'
+require 'uri'
+require 'rubygems/installer'
+require 'allgems/Indexer'
+
+module AllGems
+    class GemWorker
+        class << self
+            attr_reader :pool
+            # Get the worker ready to go
+            def setup
+                @glock = Mutex.new
+                @slock = Mutex.new
+                @pool = ActionPool::Pool.new(:max_threads => 10, :a_to => 60*5)
+            end
+            # :name:: name of the gem
+            # :version:: version of the gem
+            # :database:: database connection
+            def process(args)
+                AllGems.logger.info "Processing gem: #{args[:name]}-#{args[:version]}"
+                spec,uri = self.get_spec(args[:name], args[:version])
+                raise NameError.new("Name not found: #{args[:name]} - #{args[:version]}") if spec.nil?
+                basedir = "#{AllGems.data_directory}/#{spec.name}/#{spec.version.version}"
+                FileUtils.mkdir_p "#{basedir}/unpack"
+                AllGems.logger.info "Created new directory: #{basedir}/unpack"
+                gempath = self.fetch(spec, uri, "#{basedir}/#{spec.full_name}.gem")
+                self.unpack(gempath, basedir)
+                self.generate_documentation(spec, basedir)
+                self.save_data(spec, args[:database])
+            end
+
+            # name:: name of gem
+            # version:: version of gem
+            # Fetches the Gem::Specification for the given gem
+            def get_spec(name, version)
+                AllGems.logger.info "Fetching gemspec for #{name}-#{version}"
+                dep = Gem::Dependency.new(name, version)
+                spec = nil
+                @glock.synchronize{spec = Gem::SpecFetcher.fetcher.fetch dep, true}
+                spec[0]
+            end
+
+            # spec:: Gem::Specification
+            # uri:: URI of gem files home
+            # save_path:: path to save gem
+            # Fetch the gem file from the server. Returns the path to the gem
+            # on the local machine
+            def fetch(spec, uri, save_path)
+                AllGems.logger.info "Fetching gem from: #{uri}/gems/#{spec.full_name}.gem"
+                FileUtils.touch(save_path)
+                begin
+                    remote_path = "#{uri}/gems/#{spec.full_name}.gem"
+                    remote_uri = URI.parse(remote_path)
+                    file = File.open(save_path, 'wb')
+                    file.write(self.fetch_remote(remote_uri))
+                    file.close
+                    save_path
+                rescue StandardError => boom
+                    raise FetchError.new(spec.name, spec.version, remote_path, boom)
+                end
+            end
+
+            # uri:: URI of gem
+            # depth:: number of times called
+            # Fetch gem from given URI
+            def fetch_remote(uri, depth=0)
+                raise IOError.new("Depth too deep") if depth > 9
+                response = Net::HTTP.get_response(uri)
+                if(response.is_a?(Net::HTTPSuccess))
+                    response.body
+                elsif(response.is_a?(Net::HTTPRedirection))
+                    self.fetch_remote(URI.parse(response['location']), depth + 1)
+                else
+                    raise IOError.new("Unknown response type: #{response}")
+                end
+            end
+
+            # gempath:: path to gem file
+            # newname:: path to move gem file to
+            # Moves the gem to the given location
+            def save(gempath, newpath)
+                AllGems.logger.info "Moving #{gempath} to #{newpath}"
+                FileUtils.mv gempath, newpath
+                newpath
+            end
+
+            # path:: path to the gem file
+            # basedir:: directory to unpack in
+            # depth:: number of times called
+            # Unpacks the gem into the basedir under the 'unpack' directory
+            def unpack(path, basedir, depth=0)
+                AllGems.logger.info "Unpacking gem: #{path}"
+                begin
+                    Gem::Installer.new(path, :unpack => true).unpack "#{basedir}/unpack"
+                    FileUtils.chmod_R(0755, "#{basedir}/unpack") # fix any bad permissions
+                    FileUtils.rm(path)
+                rescue
+                    if(File.size(path) < 1 || depth > 10)
+                        raise IOError.new("Failed to unpack gem: #{path}") unless self.direct_unpack(path, basedir)
+                    else
+                        self.unpack(path, basedir, depth+1)
+                    end
+                end
+                true
+            end
+
+            # path:: path to the gem file
+            # basedir:: directory to unpack in
+            # Last ditch effort to unpack the gem
+            def direct_unpack(path, basedir)
+                AllGems.logger.warn "Attempting forcible unpack on: #{path}"
+                unpackdir = path.slice(0, path.rindex('.'))
+                self.run_command("cd #{basedir} && gem unpack #{path} && mv #{unpackdir}/* #{basedir}/unpack/ && rm -rf #{unpackdir}")
+            end
+
+            # dir:: base directory location of gem contents
+            # Generates the documentation of the given directory of ruby code. Appends
+            # 'unpack' to the given directory for code discovery. Documentation will
+            # be output in "#{dir}/doc"
+            def generate_documentation(spec, dir)
+                AllGems.logger.info "Generating documentation for #{spec.full_name}"
+                AllGems.doc_format.each do |f|
+                    command = nil
+                    args = []
+                    case f.to_sym
+                        when :rdoc
+                            command = 'rdoc'
+                            args << '--format=darkfish' << '-aFNqH' << "--op=#{dir}/doc/rdoc" << "#{dir}/unpack"
+                        when :sdoc
+                            command = 'sdoc'
+                            args << '-T direct' << "-o #{dir}/doc/sdoc" << "#{dir}/unpack"
+                        when :hanna
+                            command = "ruby #{AllGems.hanna_hack}"
+                            args << "-o #{dir}/doc/hanna" << "#{dir}/unpack"
+                        else
+                            next # if we don't know what to do with it, skip it
+                    end
+                    action = lambda do |spec, dir, command, args, f|
+                        FileUtils.rm_r("#{dir}/doc/#{f}", :force => true) # make sure we are clean before we get dirty
+                        result = self.run_command("#{command} #{args}")
+                        raise DocError.new(spec.name, spec.version) unless result
+                        AllGems.logger.info "Completed documentation for #{spec.full_name}"
+                        FileUtils.chmod_R(0755, "#{dir}/doc/#{f}") # fix any bad permissions
+                        Indexer.index_gem(spec, "#{dir}/doc/#{f}", f)
+                    end
+                    @pool << [action, [spec, dir.dup, command.dup, args.join(' '), f]]
+                end
+            end
+
+            # command:: command to run
+            # return_output:: return output
+            # Runs a command. Returns true if status returns 0. If return_output is true,
+            # return value is: [status, output]
+            def run_command(command, return_output=false)
+                AllGems.logger.debug "Command to be executed: #{command}"
+                pro = nil
+                output = []
+                status = nil
+                begin
+                    pro = IO.popen(command)
+                    Process.setpriority(Process::PRIO_PROCESS, pro.pid, 19) unless ON_WINDOWS
+                    until(pro.closed? || pro.eof?)
+                        output << pro.gets
+                    end
+                ensure
+                    unless(pro.nil?)
+                        pid, status = Process.waitpid2(pro.pid)
+                    else
+                        status = 1
+                    end
+                end
+                return return_output ? [status == 0, output.join] : status == 0
+            end
+
+            # spec:: Gem::Specification
+            # Save data to the database about this gem
+            def save_data(spec, db)
+                AllGems.logger.info "Saving meta data for #{spec.full_name}"
+                @slock.synchronize do
+                    gid = db[:gems].filter(:name => spec.name).first
+                    gid = gid.nil? ? db[:gems].insert(:name => spec.name) : gid[:id]
+                    pid = db[:platforms].filter(:platform => spec.platform).first
+                    pid = pid.nil? ? db[:platforms].insert(:platform => spec.platform) : pid[:id]
+                    vid = db[:versions] << {:version => spec.version.version, :gem_id => gid, :platform_id => pid, :release => spec.date}
+                    db[:specs] << {:version_id => vid, :spec => [Marshal.dump(spec)].pack('m')}
+                    db[:gems].filter(:id => gid).update(:summary => spec.summary)
+                    db[:gems].filter(:id => gid).update(:description => spec.description)
+                end
+                AllGems.logger.info "Meta data saving complete for #{spec.full_name}"
+                true
+            end
+
+            # Make a parent error class that we can specialize
+            class Error < StandardError
+                attr_reader :original
+                def initialize(e=nil)
+                    @original = e.nil? ? self : e
+                end
+            end
+
+            # Exception class for failed documentation creation
+            class DocError < Error
+                attr_reader :gem_name, :gem_version
+                def initialize(gn, gv, e=nil)
+                    super(e)
+                    @gem_name = gn
+                    @gem_version = gv
+                end
+                def to_s
+                    "Failed to create documentation for: #{@gem_name}-#{@gem_version}."
+                end
+            end
+
+            # Exception class for failed gem fetching
+            class FetchError < Error
+                attr_reader :gem_name, :gem_version, :uri
+                def initialize(gn, gv, u, e=nil)
+                    super(e)
+                    @gem_name = gn
+                    @gem_version = gv
+                    @uri = u
+                end
+                def to_s
+                    "Failed to fetch #{@gem_name}-#{@gem_version}.gem from #{uri}"
+                end
+            end
+        end
+    end
+end