gem_velocity 0.0.10 → 0.0.11

data/TODO

@@ -1,16 +1,5 @@
-Add a Version?
-
-currently, the versions in the base class are single versions, but they are essentially used for representing a specific line
-
-in default_line_datas.
-(and then used for titles, abd bs).
-
-if rubygems 606 isn't solved, why not just put a total on there and let the inconsistnecies just be with some notice?
-
 rename time_format_str_small alias
 
-swap out gruff_builder to some other composing object of a builder, that can mash up the various velocitators?
-
 consolidate velocitator fixtrues or at least repeatable generators in specs.
 
 cassandra says to have an option to have the multiplexer start them from the same data (to see curve/adoption rate). I see it.
@@ -25,4 +14,4 @@ speed the fuck out of it:
   Wrote graph to /Users/shair/projects/gem_velocities2/gem_velocity_web/public/images/AggregatedVelocitator-rails-3.2.x.png
   2013-10-27 17:46:53 -0700
 
-add total that is displayed on rubygems (vs total that it gets back and calculates per day).
+use the perftool stuff.

data/benchmark/bench.rb

@@ -0,0 +1,64 @@
+#!/usr/bin/env ruby
+
+BENCH_DIR = File.expand_path(File.dirname(__FILE__))
+project = File.expand_path(File.join(BENCH_DIR, ".."))
+Dir.chdir project
+$:.unshift(project)
+$:.unshift("lib")
+
+require File.join(project,"lib", "gem_velocity")
+
+require 'benchmark'
+require 'perftools'
+
+class PerfToolHelper
+  class << self
+
+    def clean_slate
+      FileUtils.rm_rf(tmp_dir)
+      FileUtils.mkdir_p(tmp_dir)
+    end
+
+    def tmp_dir
+      "#{BENCH_DIR}/tmp"
+    end
+
+    attr_reader :paths
+    def gather_data(path, &block)
+      @paths ||= []
+      @paths << path
+      PerfTools::CpuProfiler.start(path)
+      block.call
+      PerfTools::CpuProfiler.stop
+    end
+  end
+end
+
+puts 'cleaning previous perftool'
+PerfToolHelper.clean_slate
+
+Benchmark.bm do |b|
+    PerfToolHelper.gather_data("#{PerfToolHelper.tmp_dir}/2.3.x") do
+      b.report("aggregated") { BaseVelocitator.create(:gem_name => "rails", :version => "2.3.x").graph }
+    end
+end
+
+Benchmark.bm do |b|
+    PerfToolHelper.gather_data("#{PerfToolHelper.tmp_dir}/0.0.x") do
+      b.report("single") { BaseVelocitator.create(:gem_name => "haml-i18n-extractor", :version => "0.0.5").graph }
+    end
+end
+
+puts 'multi'
+puts Benchmark.measure {
+  one = BaseVelocitator.create(:gem_name => "haml-i18n-extractor", :version => "0.0.16" )
+  two = BaseVelocitator.create(:gem_name => "haml-i18n-extractor", :version => "0.0.17" )
+  PerfToolHelper.gather_data("#{PerfToolHelper.tmp_dir}/0.0.16-0.0.17") do
+    Multiplexer.new([one,two]).graph
+  end
+}
+
+puts "Creating images for perftools..."
+PerfToolHelper.paths.each do |perftool_data_path|
+  system("pprof.rb --gif #{perftool_data_path} > #{perftool_data_path}.gif")
+end

data/bin/gem_velocity

@@ -0,0 +1,30 @@
+#!/usr/bin/env ruby
+
+require "gem_velocity"
+
+option_parser = Trollop::Parser.new do
+  version "Current version: #{GemVelocity::VERSION}"
+  opt :gem_name, "gem name", :short => 'g', :type => String
+  opt :version_str, "version_str", :short => 'n', :type => String
+  # TODO start,end,max,min.
+  # TODO handle commas and change to multiplexer ?
+  banner <<-EOB
+
+  Run like:
+
+    gem_velocity -g <gem_name> -n <version_number>
+    # example
+    # gem_velocity -g haml-i18n-extractor -n 0.0.5
+  EOB
+end
+
+opts = Trollop::with_standard_exception_handling option_parser do
+  raise Trollop::HelpNeeded if ARGV.empty?
+  option_parser.parse ARGV
+end
+
+opts[:version] = opts.delete(:version_str)
+
+velocitator = BaseVelocitator.create(opts)
+file = velocitator.graph
+

data/gem_velocity.gemspec

@@ -21,6 +21,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "gruff"
   spec.add_dependency "activesupport"
   spec.add_dependency "gems"
+  spec.add_dependency "trollop", "1.16.2"
 
   spec.add_development_dependency "bundler", "~> 1.3"
   spec.add_development_dependency "rake"
@@ -31,5 +32,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "timecop"
   spec.add_development_dependency "sourcify"
   spec.add_development_dependency "ParseTree"
+  spec.add_development_dependency "perftools.rb"
 
 end

data/lib/gem_velocity/gem_data.rb

@@ -2,12 +2,47 @@ require 'gems'
 
 class GemData
 
+  require 'thread'
+
+  include Helpers
   attr_reader :gem_name
 
   def initialize(gem_name)
     @gem_name = gem_name
   end
 
+  def parallel_fetch_for(versions)
+    @mutex1 = Mutex.new
+    @cached_downloads_metatada ||= {}
+    smallest_version = versions.map{|v| ComparableVersion.new(v)}.sort.first.str
+    start_t = time_built(smallest_version)
+    end_t = Time.now
+    puts "fetching #{versions.size} download data requests"
+    threads = []
+
+    # TODO limit threads.
+    versions.map do |v|
+      threads << Thread.new {
+        fetch_downloads_metadata(v,start_t,end_t,@cached_downloads_metatada)
+      }
+    end
+    threads.map(&:join)
+  end
+
+  def fetch_downloads_metadata(version, start_time, end_time, cache_hash)
+    key = key_for(version, start_time, end_time)
+    data_received = Gems.downloads(gem_name, version, start_time, end_time)
+    @mutex1.synchronize do
+      # shared hash
+      cache_hash[key] ||= data_received
+    end
+  end
+
+  def prefetched_downloads_metadata(version, start_time, end_time)
+    key = key_for(version, start_time, end_time)
+    @cached_downloads_metatada[key]
+  end
+
   def versions
     versions_metadata.map{|v| v["number"]}
   end
@@ -18,8 +53,11 @@ class GemData
     h
   end
 
+  def time_built(v)
+    versions_built_at[v]
+  end
+
   def versions_metadata
-    # cache api call.
     @versions_metadata ||= Gems.versions(gem_name)
     # it should be a hash
     if @versions_metadata.is_a?(String)
@@ -30,6 +68,10 @@ class GemData
     @versions_metadata
   end
 
+  def key_for(v,s,e)
+    "#{v}-#{time_format_str_small(s)}-#{time_format_str_small(e)}"
+  end
+
   def total_for_version(version)
     @total_for_version ||= {}
     key = "#{version}"
@@ -37,13 +79,5 @@ class GemData
     @total_for_version[key] ||= Gems.total_downloads(gem_name, version)
   end
 
-  def downloads_metadata(version, start_time, end_time)
-    # cache api call.
-    @downloads_metadata ||= {}
-    key = "#{version}-#{start_time}-#{end_time}"
-    return @downloads_metadata[key] if @downloads_metadata[key]
-    @downloads_metadata[key] ||= Gems.downloads(gem_name, version, start_time, end_time).to_a
-  end
-
 
 end

data/lib/gem_velocity/helpers.rb

@@ -37,3 +37,25 @@ module Helpers
   end
 
 end
+
+# http://stackoverflow.com/questions/2051229/how-to-compare-versions-in-ruby
+class ComparableVersion < Array
+  def initialize s
+    @str = s
+    super(s.split('.').map { |e| e.to_i })
+  end
+  def str
+    @str
+  end
+  def < x
+    (self <=> x) < 0
+  end
+  def > x
+    (self <=> x) > 0
+  end
+  def == x
+    (self <=> x) == 0
+  end
+end
+
+

data/lib/gem_velocity/velocitators/aggregated_velocitator.rb

@@ -13,14 +13,13 @@ class AggregatedVelocitator < BaseVelocitator
     @gem_name = gem_name
     @version = top_level_ver #with a wildcard/x
     @versions = @aggregated_versions = gem_data.versions.select{|v| v.match(/^#{Regexp.escape(remove_trailing_x(top_level_ver))}/) }
-    super(gem_name, @aggregated_versions)
-
     raise NoSuchVersion, "no versions found for #{@version}!" if @aggregated_versions.empty?
+    after_init
   end
 
   def default_start
-    earliest_start = @aggregated_versions.map{|v| Date.parse(time_built(v)) }.compact.min
-    time_format_str(earliest_start)
+    smallest_version = @aggregated_versions.map{|v| ComparableVersion.new(v)}.sort.first.str
+    built_at(smallest_version)
   end
 
   def default_max_value

data/lib/gem_velocity/velocitators/base_velocitator.rb

@@ -81,11 +81,12 @@ class BaseVelocitator
 
   private
 
-  def initialize(gem_name, versions)
-    @gem_name = gem_name || raise(ArgumentError, 'need a name')
-    @versions = versions || raise(ArgumentError, 'required versions')
+  def after_init
+    raise ArgumentError, "Need to pass a gem name!" if (gem_name.nil? || gem_name.empty?)
+    raise ArgumentError, "Need to pass versions!" if versions.compact.empty?
     validate_correct_gem
     validate_correct_versions
+    gem_data.parallel_fetch_for(versions)
   end
 
   def validate_correct_versions
@@ -137,19 +138,20 @@ class BaseVelocitator
   end
 
   # returns # "2013-10-10" => 45
-  def accumulated_downloads_per_day(version)
-    # downloads_metadata comes back ordered by date
+  def downloads_per_day(version)
+    @downloads_per_day ||= Hash.new
+    return @downloads_per_day[version] if @downloads_per_day[version]
     ret = Hash.new(0)
-    gem_data.downloads_metadata(version, default_start, default_end).each_cons(2) do |p,n|
+    from_api = gem_data.prefetched_downloads_metadata(version, default_start, default_end)
+    from_api.each_cons(2) do |p,n|
       #day,total pairs
       curr_total = n.last
       day = n.first
       previous_day = p.first
       ret[day] = curr_total + ret[previous_day]
     end
-    ret
+    @downloads_per_day[version] = ret
   end
-  alias :downloads_per_day :accumulated_downloads_per_day
 
   def gem_data
     # need this memoized so the gem_data memoization works for the same instance

data/lib/gem_velocity/velocitators/factory.rb

@@ -22,9 +22,9 @@ class Factory
 
   def velocitator
     if @type == :aggregated
-      AggregatedVelocitator.new(@gem_name, @version)
+      @velocitator ||= AggregatedVelocitator.new(@gem_name, @version)
     elsif @type == :single
-      SingleVelocitator.new(@gem_name, @version)
+      @velocitator ||= SingleVelocitator.new(@gem_name, @version)
     else
       raise 'no velocitor found to generate!'
     end

data/lib/gem_velocity/velocitators/multiplexer.rb

@@ -6,7 +6,9 @@ class Multiplexer < BaseVelocitator
   def initialize(velocitators)
     @velocitators = velocitators
     # TODO what if there are multiple gem_names being multiplexed?
-    super(velocitators.first.gem_name, velocitators.map(&:versions).flatten)
+    @gem_name = velocitators.first.gem_name
+    @versions = velocitators.map(&:versions).flatten
+    after_init
   end
 
   def version

data/lib/gem_velocity/velocitators/single_velocitator.rb

@@ -5,8 +5,10 @@ class SingleVelocitator < BaseVelocitator
   attr_reader :version
 
   def initialize(gem_name, version)
+    @gem_name = gem_name
     @version = version
-    super(gem_name, [version])
+    @versions = [version]
+    after_init
   end
 
   def default_start
@@ -18,7 +20,7 @@ class SingleVelocitator < BaseVelocitator
   end
 
   def default_max_value
-    accumulated_downloads_per_day(@version).map {|day,total| total}.max
+    downloads_per_day(@version).map {|day,total| total}.max
   end
 
   def line_data(start_t = nil, end_t = nil)

data/lib/gem_velocity/version.rb

@@ -1,3 +1,3 @@
 module GemVelocity
-  VERSION = "0.0.10"
+  VERSION = "0.0.11"
 end

data/lib/gem_velocity.rb

@@ -1,3 +1,4 @@
+require "trollop"
 require "gem_velocity/version"
 require "gem_velocity/helpers"
 require "gem_velocity/gem_data"
@@ -17,3 +18,7 @@ require 'date'
 
 module GemVelocity
 end
+
+# this should be used for short-lived scripts.
+# can be configurable if need be.
+GC.disable

data/spec/gem_data_spec.rb

@@ -11,7 +11,8 @@ describe GemData do
 
   it "can show download information" do
     gem_data = GemData.new("haml-i18n-extractor")
-    result = gem_data.downloads_metadata("0.0.17", gem_data.versions_built_at["0.0.17"],Time.now)
+    gem_data.parallel_fetch_for(["0.0.17"])
+    result = gem_data.prefetched_downloads_metadata("0.0.17", gem_data.versions_built_at["0.0.17"],Time.now).to_a
     result.should == [["2013-06-16", 38], ["2013-06-17", 16], ["2013-06-18", 3], ["2013-06-19", 1], ["2013-06-20", 4], ["2013-06-21", 1], ["2013-06-22", 5], ["2013-06-23", 2], ["2013-06-24", 3], ["2013-06-25", 9], ["2013-06-26", 161], ["2013-06-27", 3], ["2013-06-28", 2], ["2013-06-29", 0], ["2013-06-30", 0], ["2013-07-01", 0], ["2013-07-02", 7], ["2013-07-03", 3], ["2013-07-04", 2], ["2013-07-05", 0], ["2013-07-06", 3], ["2013-07-07", 0], ["2013-07-08", 2], ["2013-07-09", 1], ["2013-07-10", 2], ["2013-07-11", 3], ["2013-07-12", 2], ["2013-07-13", 1], ["2013-07-14", 0], ["2013-07-15", 2], ["2013-07-16", 2], ["2013-07-17", 6], ["2013-07-18", 0], ["2013-07-19", 3], ["2013-07-20", 1], ["2013-07-21", 0], ["2013-07-22", 2], ["2013-07-23", 1], ["2013-07-24", 6], ["2013-07-25", 0], ["2013-07-26", 0], ["2013-07-27", 0], ["2013-07-28", 0], ["2013-07-29", 1], ["2013-07-30", 2], ["2013-07-31", 2], ["2013-08-01", 0], ["2013-08-02", 0], ["2013-08-03", 0], ["2013-08-04", 0], ["2013-08-05", 2], ["2013-08-06", 0], ["2013-08-07", 0], ["2013-08-08", 3], ["2013-08-09", 0], ["2013-08-10", 0], ["2013-08-11", 0], ["2013-08-12", 1], ["2013-08-13", 2], ["2013-08-14", 1], ["2013-08-15", 2], ["2013-08-16", 0], ["2013-08-17", 1], ["2013-08-18", 0], ["2013-08-19", 0], ["2013-08-20", 0], ["2013-08-21", 1], ["2013-08-22", 2], ["2013-08-23", 4], ["2013-08-24", 0], ["2013-08-25", 1], ["2013-08-26", 0], ["2013-08-27", 2], ["2013-08-28", 0], ["2013-08-29", 2], ["2013-08-30", 0], ["2013-08-31", 0], ["2013-09-01", 2], ["2013-09-02", 2], ["2013-09-03", 1], ["2013-09-04", 0], ["2013-09-05", 2], ["2013-09-06", 2], ["2013-09-07", 0], ["2013-09-08", 0], ["2013-09-09", 1], ["2013-09-10", 0], ["2013-09-11", 0], ["2013-09-12", 5], ["2013-09-13", 0], ["2013-09-14", 0], ["2013-09-15", 0], ["2013-09-16", 1], ["2013-09-17", 0], ["2013-09-18", 0], ["2013-09-19", 1], ["2013-09-20", 1]]
   end
 

data/spec/single_velocitator_spec.rb

@@ -15,7 +15,7 @@ describe "a Velocitator rendering graphs" do
     # should this be the specific date range values?? aggregated since when? not aggregated?
     velocitator.graph_options[:line_datas].should == [[303, 303, 303, 304, 304]]
 
-    velocitator.graph_options[:title].should == "haml-i18n-extractor-0.0.17\n(downloads: 377)"
+    velocitator.graph_options[:title].should == "haml-i18n-extractor-0.0.17\n(downloads: 386)"
     velocitator.graph_options[:labels].should == ({1=>"2013-09-13", (velocitator.line_data.size-2) =>"2013-09-17"})
     velocitator.graph_options[:max_value].should == 306 # the max in the range (time.now)
     velocitator.graph_options[:min_value].should == 0
@@ -60,7 +60,7 @@ describe SingleVelocitator do
 
   it "holds the totals of the gem" do
     velocitator = SingleVelocitator.new("haml-i18n-extractor", "0.0.17")
-    velocitator.totals_map_by_version["0.0.17"][:version_downloads].should eq 377
+    velocitator.totals_map_by_version["0.0.17"][:version_downloads].should eq 386
     #binding.pry
   end
 

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: gem_velocity
 version: !ruby/object:Gem::Version 
-  hash: 11
+  hash: 9
   prerelease: 
   segments: 
   - 0
   - 0
-  - 10
-  version: 0.0.10
+  - 11
+  version: 0.0.11
 platform: ruby
 authors: 
 - Shai Rosenfeld
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2013-10-28 00:00:00 Z
+date: 2013-10-30 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: gruff
@@ -60,9 +60,25 @@ dependencies:
   type: :runtime
   version_requirements: *id003
 - !ruby/object:Gem::Dependency 
-  name: bundler
+  name: trollop
   prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - "="
+      - !ruby/object:Gem::Version 
+        hash: 83
+        segments: 
+        - 1
+        - 16
+        - 2
+        version: 1.16.2
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: bundler
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ~>
@@ -73,11 +89,11 @@ dependencies:
         - 3
         version: "1.3"
   type: :development
-  version_requirements: *id004
+  version_requirements: *id005
 - !ruby/object:Gem::Dependency 
   name: rake
   prerelease: false
-  requirement: &id005 !ruby/object:Gem::Requirement 
+  requirement: &id006 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -87,11 +103,11 @@ dependencies:
         - 0
         version: "0"
   type: :development
-  version_requirements: *id005
+  version_requirements: *id006
 - !ruby/object:Gem::Dependency 
   name: rspec
   prerelease: false
-  requirement: &id006 !ruby/object:Gem::Requirement 
+  requirement: &id007 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -101,11 +117,11 @@ dependencies:
         - 0
         version: "0"
   type: :development
-  version_requirements: *id006
+  version_requirements: *id007
 - !ruby/object:Gem::Dependency 
   name: pry
   prerelease: false
-  requirement: &id007 !ruby/object:Gem::Requirement 
+  requirement: &id008 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -115,11 +131,11 @@ dependencies:
         - 0
         version: "0"
   type: :development
-  version_requirements: *id007
+  version_requirements: *id008
 - !ruby/object:Gem::Dependency 
   name: vcr
   prerelease: false
-  requirement: &id008 !ruby/object:Gem::Requirement 
+  requirement: &id009 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -129,11 +145,11 @@ dependencies:
         - 0
         version: "0"
   type: :development
-  version_requirements: *id008
+  version_requirements: *id009
 - !ruby/object:Gem::Dependency 
   name: webmock
   prerelease: false
-  requirement: &id009 !ruby/object:Gem::Requirement 
+  requirement: &id010 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - "="
@@ -144,11 +160,11 @@ dependencies:
         - 13
         version: "1.13"
   type: :development
-  version_requirements: *id009
+  version_requirements: *id010
 - !ruby/object:Gem::Dependency 
   name: timecop
   prerelease: false
-  requirement: &id010 !ruby/object:Gem::Requirement 
+  requirement: &id011 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -158,11 +174,11 @@ dependencies:
         - 0
         version: "0"
   type: :development
-  version_requirements: *id010
+  version_requirements: *id011
 - !ruby/object:Gem::Dependency 
   name: sourcify
   prerelease: false
-  requirement: &id011 !ruby/object:Gem::Requirement 
+  requirement: &id012 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -172,11 +188,11 @@ dependencies:
         - 0
         version: "0"
   type: :development
-  version_requirements: *id011
+  version_requirements: *id012
 - !ruby/object:Gem::Dependency 
   name: ParseTree
   prerelease: false
-  requirement: &id012 !ruby/object:Gem::Requirement 
+  requirement: &id013 !ruby/object:Gem::Requirement 
     none: false
     requirements: 
     - - ">="
@@ -186,12 +202,26 @@ dependencies:
         - 0
         version: "0"
   type: :development
-  version_requirements: *id012
+  version_requirements: *id013
+- !ruby/object:Gem::Dependency 
+  name: perftools.rb
+  prerelease: false
+  requirement: &id014 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 3
+        segments: 
+        - 0
+        version: "0"
+  type: :development
+  version_requirements: *id014
 description: generate gem velocity images
 email: 
 - srosenfeld@engineyard.com
-executables: []
-
+executables: 
+- gem_velocity
 extensions: []
 
 extra_rdoc_files: []
@@ -203,6 +233,8 @@ files:
 - README.md
 - Rakefile
 - TODO
+- benchmark/bench.rb
+- bin/gem_velocity
 - examples/README.md
 - examples/generate_images.rb
 - examples/public/images/AggregatedVelocitator-haml-i18n-extractor-0.4.x.png