gem_stats 0.0.1 → 0.0.2

data/.gitignore

@@ -2,3 +2,4 @@
 .bundle
 Gemfile.lock
 pkg/*
+tmtags

data/Rakefile

@@ -1,5 +1,4 @@
 # encoding: UTF-8
-
 require "bundler/gem_tasks"
 require 'rake/testtask'
 

data/bin/fetchgemstats

@@ -1,7 +1,21 @@
 #!/usr/bin/env ruby
 require 'gem_stats'
+require 'optparse'
 
-puts 'Starting fetching...'
 start = Time.now
+
+options = {:config => {'development' => {}, 'production' => {}}, :env => 'development'}
+
+OptionParser.new do |opts|
+  opts.banner = 'Usage: fetchgemstats [options]'
+  opts.on('-e', '--env [environment]', 'Default is development environment') { |opt| options[:env] = 'production' }
+  opts.on('-u', '--uri [mongo db URI]',
+    'Pattern: mongodb://[username:password@]host1[:port1][,host2[:port2],...[,hostN[:portN]]][/[database][?options]]') { |opt| options[:config][options[:env]]['uri'] = opt }
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts opts
+    exit
+  end
+end.parse!
+
+GemStats::Connection.connect(options[:config][options[:env]].empty? ? {} : options)
 GemStats::Fetcher.fetch_and_safe
-puts "Finished in #{(Tine.now - start).round(0)} second(s)"

data/config/database.yml

@@ -0,0 +1,2 @@
+development:
+  uri: mongodb://localhost:27017/gem_stats

data/gem_stats.gemspec

@@ -23,4 +23,5 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency "json"
   s.add_runtime_dependency "mongo_mapper"
   s.add_runtime_dependency "bson_ext"
+  s.add_runtime_dependency "ruby-progressbar"
 end

data/lib/gem_stats/connection.rb

@@ -1,8 +1,19 @@
+require 'erb'
+
 module GemStats
   module Connection
-    def self.connect
-      MongoMapper.connection = Mongo::Connection.new('localhost')
-      MongoMapper.database = 'gem_stats'
+    def self.connect(options = {})
+      if options.empty?
+        env, path = defined?(Rails)? [Rails.env, [Rails.root, 'config/database.yml']] :
+          [ENV['ENV'] || 'development', File.expand_path('../../../config/database.yml', __FILE__)]
+        get_config(path) {|config| MongoMapper.setup(config, env) }
+      else
+        MongoMapper.setup(options[:config], options[:env])
+      end
+    end
+
+    def self.get_config(*path)
+      yield YAML::load(ERB.new(File.read File.join(path)).result)
     end
   end
 end

data/lib/gem_stats/fetcher.rb

@@ -1,24 +1,34 @@
 require 'rubygems/spec_fetcher'
+require 'progressbar'
 
 module GemStats
   class Fetcher
+    @@bulk_number = 50
+    cattr_accessor :bulk_number
+
     def self.list(all_versions = false, prerelease = false)
       @gem_list ||= ::Gem::SpecFetcher.new.list(all_versions, prerelease).values.flatten(1)
     end
 
     def self.list_names
-      @list_name ||= list.map(&:first)
+      @list_name ||= list.map(&:first).uniq.compact
     end
 
     def self.fetch_and_safe
+      pbar = ProgressBar.new("Fetching gems", list.count)
+      pbar.bar_mark = '#'
       threads = []
-      list_names.each do |name|
+      list_names.each_with_index do |name, index|
         threads << Thread.new(name) { |name| GemInfo.new(name).save }
-        if threads.count == 50
+        if threads.count == @@bulk_number
           threads.each(&:join)
           threads = []
+          pbar.inc(bulk_number)
         end
       end
+      threads.each(&:join)
+      pbar.inc(bulk_number)
+      pbar.finish
     end
   end
 end

data/lib/gem_stats/gem.rb

@@ -5,8 +5,7 @@ module GemStats
     include MongoMapper::Document
 
     key :name, String
-    key :runtime_dependencies, Array
-    key :development_dependencies, Array
+    key :dependencies, Hash
     key :version, String
     key :downloads, Integer
     key :version_downloads, String
@@ -20,5 +19,24 @@ module GemStats
     key :mailing_list_uri, String
     key :source_code_uri, String
     key :bug_tracker_uri, String
+
+    # to generate pretty url for rails
+    def to_param
+      name
+    end
+
+    def gem_info
+      @gem_info ||= GemInfo.new(name)
+    end
+
+    %w(runtime development).each do |env|
+      define_method "#{env}_dependencies" do
+        dependencies[env]
+      end
+
+      define_method "#{env}_dependents" do
+        eval "@#{env}_dependents ||= gem_info.clients('#{env}')"
+      end
+    end
   end
 end

data/lib/gem_stats/gem_info.rb

@@ -13,42 +13,31 @@ module GemStats
     end
 
     def info
-      @info ||= JSON.load open("#{API_URL}#{@name}.#{REQUEST_FORMAT}").read
+      return @info if defined?(@info)
+      begin
+        @info ||= JSON.load open("#{API_URL}#{@name}.#{REQUEST_FORMAT}").read
+      rescue OpenURI::HTTPError => e
+        @info = nil
+      end
     end
 
     def dependencies(env = 'runtime')
       @dependencies ||= {}
-      @dependencies[env.to_s] ||= info['dependencies'][env.to_s].inject([]) { |sum, gem| sum << gem['name'] }
+      @dependencies[env.to_s] ||= info['dependencies'][env.to_s].inject([]) { |list, gem| list << gem['name'] }
     end
 
     def clients(env = 'runtime')
-      Gem.where("#{env}_dependencies" => @name).all
+      Gem.where("dependencies.#{env}.name" => @name).all
     end
 
     def save
-      log @name
       gem = Gem.find_by_name(@name) || Gem.new
-      gem.attributes = {:name                     => @name,
-                        :runtime_dependencies     => dependencies,
-                        :development_dependencies => dependencies('development'),
-                        :version                  => info['version'],
-                        :downloads                => info['downloads'],
-                        :version_downloads        => info['version_downloads'],
-                        :authors                  => info['authors'],
-                        :info                     => info['info'],
-                        :project_uri              => info['project_uri'],
-                        :gem_uri                  => info['gem_uri'],
-                        :homepage_uri             => info['homepage_uri'],
-                        :wiki_uri                 => info['wiki_uri'],
-                        :documentation_uri        => info['documentation_uri'],
-                        :mailing_list_uri         => info['mailing_list_uri'],
-                        :source_code_uri          => info['source_code_uri'],
-                        :bug_tracker_uri          => info['bug_tracker_uri']}
-      gem.save
-    end
-
-    def log(message)
-      puts message
+      if info
+        gem.attributes = info
+        gem.save
+      else
+        false
+      end
     end
   end
 end

data/lib/gem_stats/version.rb

@@ -1,3 +1,3 @@
 module GemStats
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/lib/gem_stats.rb

@@ -6,6 +6,3 @@ require "gem_stats/connection"
 
 module GemStats
 end
-
-GemStats::Connection.connect
-

data/test/fetcher_test.rb

@@ -7,11 +7,13 @@ class FetcherTest < Test::Unit::TestCase
     assert_not_nil Fetcher.list
   end
 
-  def test_size_of_fetcher_list
-    assert_equal Fetcher.list.count, Fetcher.list_names.count
-  end
-
   def test_fetch_and_save
     assert_equal true, Fetcher.respond_to?(:fetch_and_safe)
   end
+
+  def test_bulk_number
+    assert_equal 50, Fetcher.bulk_number
+    Fetcher.bulk_number = 100
+    assert_equal 100, Fetcher.bulk_number
+  end
 end

data/test/gem_info_test.rb

@@ -2,6 +2,7 @@ require 'test_helper'
 
 class GemInfoTest < Test::Unit::TestCase
   def setup
+    GemStats::Connection.connect
     @gem = GemStats::GemInfo.new('get_or_build')
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: gem_stats
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-03-29 00:00:00.000000000Z
+date: 2012-07-02 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
-  requirement: &70259170946480 !ruby/object:Gem::Requirement
+  requirement: &70130075293200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70259170946480
+  version_requirements: *70130075293200
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70259170946020 !ruby/object:Gem::Requirement
+  requirement: &70130075292220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70259170946020
+  version_requirements: *70130075292220
 - !ruby/object:Gem::Dependency
   name: mongo_mapper
-  requirement: &70259170945420 !ruby/object:Gem::Requirement
+  requirement: &70130075290980 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70259170945420
+  version_requirements: *70130075290980
 - !ruby/object:Gem::Dependency
   name: bson_ext
-  requirement: &70259170944820 !ruby/object:Gem::Requirement
+  requirement: &70130075290060 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,7 +54,18 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70259170944820
+  version_requirements: *70130075290060
+- !ruby/object:Gem::Dependency
+  name: ruby-progressbar
+  requirement: &70130075288840 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70130075288840
 description: Just put gemstats your_gem_name and have a pleasure to see full statistic
   for this gem
 email:
@@ -70,7 +81,7 @@ files:
 - Rakefile
 - bin/fetchgemstats
 - bin/geminfo
-- config/mongo.yml
+- config/database.yml
 - gem_stats.gemspec
 - lib/gem_stats.rb
 - lib/gem_stats/connection.rb
@@ -95,7 +106,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -2815935118913676328
+      hash: -402065578050743399
 required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
   requirements:
@@ -104,7 +115,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
       segments:
       - 0
-      hash: -2815935118913676328
+      hash: -402065578050743399
 requirements: []
 rubyforge_project: gem_stats
 rubygems_version: 1.8.10

data/config/mongo.yml

@@ -1,2 +0,0 @@
-host: localhost
-database: gem_stats