github_archive_parser 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: dbbcc01faaf08baed69942f2c59ffe2fca262966
-  data.tar.gz: 4aaba6c19c5caa3e2a4b17fa331e2a0411cac792
+  metadata.gz: cc577ab9b44ecf0384854bcabef6d03407beb841
+  data.tar.gz: 3e0af6d7aab095936d5865bc9f9b538cc6e60cd0
 SHA512:
-  metadata.gz: f4e608379a73168cf4de92cd2ef11e62fb022f3f778afa43d57dd3508e26d9b188cc26114d86a5cac4b55c8638a856545a0e3ce2f525f17abdd19c633feb9cf3
-  data.tar.gz: 08d9faf62bdad2358c01d2f65440b8812fc91965ecc4c16f08af6fc98fb5dd4703914d43c9541b31d9b21ac7e4459b7b99ef1d7e7c2e4b2a9bf752eb3f0f9de9
+  metadata.gz: 253ce14d775c224434c7a498db8545ac9e4e6fd2be77cf8ed429280fabfb923b25c87a2a60787c279f80903a13e520342664859f92d14f4aa2e825a5912ed874
+  data.tar.gz: 5a479e595bf17b801e89b0af6293383161817585b6b35949e7aa327c16a666a64f5652c0761419ed438928b0e4279d702f31a916d5cfac9557d21c727f1dff0c

data/bin/github_archive_parser

@@ -3,7 +3,5 @@
 $:.unshift File.expand_path("../../lib", __FILE__)
 require 'github_archive_parser'
 
-github_archive_parser = GitHubArchiveParser::Processor.new
-ARGV.each do |url|
-  github_archive_parser.process(url)
-end
+github_archive_parser = GitHubArchiveParser::CLI.new
+github_archive_parser.process(ARGV)

data/lib/github_archive_parser.rb

@@ -1,42 +1,37 @@
 require 'github_archive_parser/initialize'
 
 module GitHubArchiveParser
-  class Processor
-    attr_reader :options
+  class CLI
+    attr_accessor :options
 
     def initialize
       @options = OpenStruct.new(
         debug: false,
+        quite: false,
+        since: nil,
+        until: nil,
       )
       parse_options
       determine_log_level
-
-      # Create the concrete handlers and store them for future use
-      create_event_handlers
     end
 
-    def process(url)
-      Log.info "Processing #{url}"
-      if (!url.nil? && url.start_with?("http://data.githubarchive.org/"))
-        gz = open(url)
-        js = Zlib::GzipReader.new(gz).read
-
-        Yajl::Parser.parse(js) do |event|
-          event = Hashie::Mash.new(event)
-          event_class = class_from_string("GitHubArchiveParser::#{event.type}")
-          event_handler = @event_handlers[event_class]
-
-          event_handler.each { |handler|
-            if !handler.nil? && handler.respond_to?(:parse)
-              handler.parse(event)
-            end
-          }
+    def process(args)
+      processor = Processor.new
+      begin
+        if !@options.until.nil? && !@options.since.nil?
+          processor.process_between(@options.since, @options.until)
+        elsif !@options.since.nil?
+          processor.process_since(@options.since)
+        else
+          args.each {|url| processor.process_url(url) }
         end
-      else
-        Log.warn "URL[#{url}] does not belong to http://data.githubarchive.org/"
+      rescue Exception => e
+        Log.error e
       end
     end
 
+    private
+
     def parse_options
       OptionParser.new do |opt|
         opt.version = VERSION
@@ -46,11 +41,15 @@ module GitHubArchiveParser
         opt.on "-q", "--quite", "Hide all output (shows only UNKNOWN level log statements)" do
           options.quite = true
         end
+        opt.on "-s", "--since TIME", "Process all events since the provided date (can be specified in natural language)" do |time|
+          options.since = time
+        end
+        opt.on "-u", "--until TIME", "Process all events until the provided date (can be specified in natural language) starting from February 12, 2011" do |time|
+          options.until = time
+        end
       end.parse!
     end
 
-    private
-
     def determine_log_level
       if options.debug
         Log.level = Logger::DEBUG
@@ -62,36 +61,5 @@ module GitHubArchiveParser
       end
     end
 
-    def class_from_string(string)
-      begin
-        string.split('::').inject(Object) do |mod, class_name|
-          mod.const_get(class_name)
-        end
-      rescue Exception
-        Log.warn "Event #{string} not found"
-        nil
-      end
-    end
-
-    def create_event_handlers
-      @event_handlers = {}
-
-      # Probably can do something to not hardcode this
-      # Iterate over the event types
-      [CommitCommentEvent, CreateEvent, DeleteEvent,
-      DeploymentEvent, DeploymentStatusEvent, DownloadEvent,
-      FollowEvent, ForkApplyEvent, ForkEvent,
-      GistEvent, GollumEvent, IssueCommentEvent,
-      IssueCommentEvent, IssuesEvent, MemberEvent,
-      PublicEvent, PullRequestEvent, PullRequestReviewCommentEvent,
-      PushEvent, ReleaseEvent, StatusEvent,
-      TeamAddEvent, WatchEvent].each do | event_type |
-
-        # Map list of concrete event handler to their event type
-        @event_handlers[event_type] = event_type.descendants.map { |handler|
-          handler.new
-        }
-      end
-    end
   end
 end

data/lib/github_archive_parser/XCust.rb

@@ -2,9 +2,14 @@ module AwesomeApplication
   class PrintCreateEvent
     include GitHubArchiveParser::CreateEvent
 
+    attr_accessor :counter
+
     def parse(event)
       # The event is a Hashie::Mash object for easy (dot) access
-      puts "#{event.repository.owner}/#{event.repository.name}"
+      #puts "#{event.repository.owner}/#{event.repository.name}"
+      @counter = 0 if @counter.nil?
+      @counter = 1 + @counter
+      puts @counter
     end
   end
 end

data/lib/github_archive_parser/initialize.rb

@@ -5,5 +5,8 @@ require 'open-uri'
 require 'zlib'
 require 'yajl'
 require 'hashie'
+require 'chronic'
 
+# Require event_handler first to prevent load order issues
+require_relative './event_handler'
 Dir.glob(File.dirname(__FILE__) + '/**/*.rb') { |file| require file }

data/lib/github_archive_parser/processor.rb

@@ -0,0 +1,74 @@
+module GitHubArchiveParser
+  class Processor
+
+    def initialize
+      create_event_handlers
+    end
+
+    def process_between(since_time, until_time)
+      start_time = Utilities.time_from_natural_language(since_time)
+      end_time = Utilities.time_from_natural_language(until_time)
+      Log.info "Processing between #{start_time} and #{end_time}"
+
+      # Start hourly iterator from start time, but exclude last hour (could be incomplete)
+      iterator = start_time
+      while iterator < end_time - 3600
+        process_url("http://data.githubarchive.org/#{iterator.year}-#{iterator.month.to_s.rjust(2, '0')}-#{iterator.day.to_s.rjust(2, '0')}-#{iterator.hour}.json.gz")
+        iterator += 3600
+      end
+    end
+
+    def process_since(since_time)
+      process_between(since_time, 'Now')
+    end
+
+    def process_url(url)
+      Log.info "Processing #{url}"
+      if (!url.nil? && url.start_with?("http://data.githubarchive.org/"))
+
+        begin
+          gz = open(url)
+          js = Zlib::GzipReader.new(gz).read
+        rescue => e
+          raise e, "URL[#{url}] #{e.message}"
+        end
+
+        Yajl::Parser.parse(js) do |event|
+          event = Hashie::Mash.new(event)
+          event_class = Utilities.class_from_string("GitHubArchiveParser::#{event.type}")
+          event_handler = @event_handlers[event_class]
+
+          event_handler.each { |handler|
+            if !handler.nil? && handler.respond_to?(:parse)
+              handler.parse(event)
+            end
+          }
+        end
+      else
+        raise "URL[#{url}] does not belong to http://data.githubarchive.org/"
+      end
+    end
+
+    private
+
+    def create_event_handlers
+      @event_handlers = {}
+
+      # Probably can do something to not hardcode this
+      # Iterate over the event types
+      [CommitCommentEvent, CreateEvent, DeleteEvent,
+       DeploymentEvent, DeploymentStatusEvent, DownloadEvent,
+       FollowEvent, ForkApplyEvent, ForkEvent,
+       GistEvent, GollumEvent, IssueCommentEvent,
+       IssueCommentEvent, IssuesEvent, MemberEvent,
+       PublicEvent, PullRequestEvent, PullRequestReviewCommentEvent,
+       PushEvent, ReleaseEvent, StatusEvent,
+       TeamAddEvent, WatchEvent].each do | event_type |
+
+        # Map list of concrete event handler to their event type
+        @event_handlers[event_type] = event_type.descendants.map { |handler| handler.new }
+      end
+    end
+
+  end
+end

data/lib/github_archive_parser/utilities.rb

@@ -0,0 +1,22 @@
+module GitHubArchiveParser
+  module Utilities
+    module_function
+
+    def class_from_string(string)
+      begin
+        string.split('::').inject(Object) do |mod, class_name|
+          mod.const_get(class_name)
+        end
+      rescue Exception
+        Log.warn "Event #{string} not found"
+        nil
+      end
+    end
+
+    def time_from_natural_language(natural_language)
+      time = Chronic.parse(natural_language)
+      raise "Invalid time: #{natural_language}" if time.nil?
+      time.getutc
+    end
+  end
+end

data/lib/github_archive_parser/version.rb

@@ -1,3 +1,3 @@
 module GitHubArchiveParser
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
 end

data/spec/processor_spec.rb

@@ -0,0 +1,95 @@
+require 'spec_helper'
+
+include GitHubArchiveParser
+
+describe Processor do
+  let (:processor) { GitHubArchiveParser::Processor.new }
+  describe "#process_url" do
+    it "should fail on an invalid text" do
+      expect {processor.process_url("dummydata")}.to raise_error(RuntimeError)
+    end
+
+    it "should fail on an invalid URL" do
+      expect {processor.process_url("http://data.githubarchive.org/dummydata.json.gz")}.to raise_error(OpenURI::HTTPError)
+    end
+
+    it "should fail on an wrong domain URL" do
+      expect {processor.process_url("http://data.wrongdomain.org/dummydata.json.gz")}.to raise_error(RuntimeError)
+    end
+  end
+
+  describe "process_since" do
+    it "should call process_between(since, 'Now')" do
+      since_time = 'Yesterday'
+      until_time = 'Now'
+      processor.should receive(:process_between).with(since_time, until_time)
+      processor.process_since(since_time)
+    end
+
+    context "with invalid time" do
+      it "should fail with invalid since time" do
+        since_time = 'nil'
+        allow(processor).to receive(:process_url)
+        expect {processor.process_since(since_time)}.to raise_error(RuntimeError)
+      end
+    end
+  end
+
+  describe "process_between" do
+    it "should call process_url(url) for as many hour intervals between two times (hours)" do
+      since_time = '12 hours ago'
+      until_time = '6 hours ago'
+      processor.should receive(:process_url).exactly(6).times
+      processor.process_between(since_time, until_time)
+    end
+
+    it "should call process_url(url) for as many hour intervals between two times (days)" do
+      since_time = '3 days ago'
+      until_time = '1 day ago'
+      processor.should receive(:process_url).exactly(48).times
+      processor.process_between(since_time, until_time)
+    end
+
+    it "should call process_url(url) for as many hour intervals between two times (months)" do
+      since_time = 'April 1 2012 at 12pm'
+      until_time = 'June 1 2012 at 12pm'
+
+      # (24 hours * (April 30 days + May 31 days)) - last hour = 1463 hours
+      processor.should receive(:process_url).exactly(1463).times
+      processor.process_between(since_time, until_time)
+    end
+
+    it "should call process_url(url) for as many hour intervals between two times (years)" do
+      since_time = 'April 1 2011 at 12pm'
+      until_time = 'April 1 2013 at 12pm'
+
+      # ((Number of days between April 1 2011 and April 1 2013) * 24 hours) - last hour = 731 days * 24 hours = 17543 hours
+      processor.should receive(:process_url).exactly(17543).times
+      processor.process_between(since_time, until_time)
+    end
+
+    context "with invalid time" do
+      it "should fail with invalid since time" do
+        since_time = 'nil'
+        until_time = 'Now'
+        allow(processor).to receive(:process_url)
+        expect {processor.process_between(since_time, until_time)}.to raise_error(RuntimeError)
+      end
+
+      it "should fail with invalid until time" do
+        since_time = '1 day ago'
+        until_time = 'nil'
+        allow(processor).to receive(:process_url)
+        expect {processor.process_between(since_time, until_time)}.to raise_error(RuntimeError)
+      end
+
+      it "should fail with both invalid until/since time" do
+        since_time = 'nil'
+        until_time = 'nil'
+        allow(processor).to receive(:process_url)
+        expect {processor.process_between(since_time, until_time)}.to raise_error(RuntimeError)
+      end
+    end
+  end
+end
+

data/spec/utilities_spec.rb

@@ -0,0 +1,48 @@
+require 'spec_helper'
+
+include GitHubArchiveParser
+
+describe Utilities do
+  describe "#time_from_natural_language" do
+    it "should raise an error with an invalid time" do
+      expect { Utilities.time_from_natural_language('nil') }.to raise_error(RuntimeError)
+    end
+
+    it "should return a Time object" do
+      time = Utilities.time_from_natural_language('August 1 2013 at 4am')
+      time.should be_a(Time)
+    end
+
+    it "should return a UTC time" do
+      time = Utilities.time_from_natural_language('August 1 2013 at 4am')
+      time.utc?.should be_true
+    end
+
+    it "should return the correct time" do
+      time = Utilities.time_from_natural_language('August 1 2013 at 4am')
+      time.getlocal.year.should eq(2013)
+      time.getlocal.month.should eq(8)
+      time.getlocal.day.should eq(1)
+      time.getlocal.hour.should eq(4)
+    end
+  end
+
+  describe "#class_from_string" do
+    it "should return a valid name of a module" do
+      Utilities.class_from_string('Utilities').should eq(Utilities)
+    end
+
+    it "should return a valid name of a class" do
+      Utilities.class_from_string('Processor').should eq(Processor)
+    end
+
+    it "should return a valid name of a class using full path" do
+      Utilities.class_from_string('GitHubArchiveParser::Processor').should eq(Processor)
+    end
+
+    it "should return nil if class cannot be found" do
+      Utilities.class_from_string('SpecialApplication::MassUnit').should be_nil
+    end
+  end
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: github_archive_parser
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Kevin Jalbert
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-17 00:00:00.000000000 Z
+date: 2014-01-29 00:00:00.000000000 Z
 dependencies: []
 description: Gem which parses GitHub Archive data
 email:
@@ -46,8 +46,12 @@ files:
 - lib/github_archive_parser/event_handlers/watch_event.rb
 - lib/github_archive_parser/initialize.rb
 - lib/github_archive_parser/log.rb
+- lib/github_archive_parser/processor.rb
+- lib/github_archive_parser/utilities.rb
 - lib/github_archive_parser/version.rb
 - spec/log_spec.rb
+- spec/processor_spec.rb
+- spec/utilities_spec.rb
 homepage: https://github.com/kevinjalbert/github_archive_parser
 licenses: []
 metadata: {}
@@ -73,3 +77,5 @@ specification_version: 4
 summary: Gem which parses GitHub Archive data
 test_files:
 - spec/log_spec.rb
+- spec/processor_spec.rb
+- spec/utilities_spec.rb