hexabat 0.0.1

data/features/fixtures/single_open_issue.rb

@@ -0,0 +1,60 @@
+SINGLE_OPEN_ISSUE = %q{
+[
+  {
+    "url": "https://api.github.com/repos/path11/hexabat/issues/1",
+    "html_url": "https://github.com/path11/hexabat/issues/1",
+    "number": 1347,
+    "state": "open",
+    "title": "Found a bug",
+    "body": "I'm having a problem with this.",
+    "user": {
+      "login": "path11",
+      "id": 1,
+      "avatar_url": "https://github.com/images/error/path11_happy.gif",
+      "gravatar_id": "somehexcode",
+      "url": "https://api.github.com/users/path11"
+    },
+    "labels": [
+      {
+        "url": "https://api.github.com/repos/path11/hexabat/labels/bug",
+        "name": "bug",
+        "color": "f29513"
+      }
+    ],
+    "assignee": {
+      "login": "path11",
+      "id": 1,
+      "avatar_url": "https://github.com/images/error/path11_happy.gif",
+      "gravatar_id": "somehexcode",
+      "url": "https://api.github.com/users/path11"
+    },
+    "milestone": {
+      "url": "https://api.github.com/repos/path11/hexabat/milestones/1",
+      "number": 1,
+      "state": "open",
+      "title": "v1.0",
+      "description": "",
+      "creator": {
+        "login": "path11",
+        "id": 1,
+        "avatar_url": "https://github.com/images/error/path11_happy.gif",
+        "gravatar_id": "somehexcode",
+        "url": "https://api.github.com/users/path11"
+      },
+      "open_issues": 4,
+      "closed_issues": 8,
+      "created_at": "2011-04-10T20:09:31Z",
+      "due_on": null
+    },
+    "comments": 0,
+    "pull_request": {
+      "html_url": "https://github.com/path11/hexabat/issues/1",
+      "diff_url": "https://github.com/path11/hexabat/issues/1.diff",
+      "patch_url": "https://github.com/path11/hexabat/issues/1.patch"
+    },
+    "closed_at": null,
+    "created_at": "2011-04-22T13:33:48Z",
+    "updated_at": "2011-04-22T13:33:48Z"
+  }
+]
+}

data/features/step_definitions/callbacks.rb

@@ -0,0 +1,74 @@
+Given /^there is a single open issue on "(.*?)"$/ do |repository|
+  stub_request(:get, "https://api.github.com/repos/#{repository}/issues?page=1&per_page=100&state=open").
+    to_return(:status => 200, :body => SINGLE_OPEN_ISSUE, :headers => {})
+  stub_request(:get, "https://api.github.com/repos/#{repository}/issues?page=1&per_page=100&state=closed").
+    to_return(:status => 200, :body => '[]', :headers => {})
+end
+
+Given /^there are 101 open issues and 300 closed issues on "(.*?)"$/ do |repository|
+  #open issues page 1
+  link = '<https://api.github.com/repos/path11/hexabat/issues?page=2&per_page=100>; rel="next", <https://api.github.com/repos/path11/hexabat/issues?page=2&per_page=100>; rel="last"'
+  stub_request(:get, "https://api.github.com/repos/#{repository}/issues?page=1&per_page=100&state=open").
+    to_return(:status => 200, :body => A_100_OPEN_ISSUES, :headers => {'Link' => link})
+
+  #open issues page 2
+  link = '<https://api.github.com/repos/path11/hexabat/issues?page=1&per_page=100>; rel="last", <https://api.github.com/repos/path11/hexabat/issues?page=1&per_page=100>; rel="first", <https://api.github.com/repos/path11/hexabat/issues?page=1&per_page=100>; rel="prev"'
+  stub_request(:get, "https://api.github.com/repos/#{repository}/issues?page=2&per_page=100&state=open").
+    to_return(:status => 200, :body => SINGLE_OPEN_ISSUE, :headers => {'Link' => link})
+
+  #closed issues page 1
+  link = '<https://api.github.com/repos/path11/hexabat/issues?page=2&per_page=100&state=closed>; rel="next", <https://api.github.com/repos/path11/hexabat/issues?page=3&per_page=100&state=closed>; rel="last"'
+  stub_request(:get, "https://api.github.com/repos/#{repository}/issues?page=1&per_page=100&state=closed").
+    to_return(:status => 200, :body => CLOSED_ISSUES_PAGE_1, :headers => {'Link' => link})
+
+  #closed issues page 2
+  link = '<https://api.github.com/repos/path11/hexabat/issues?page=3&per_page=100&state=closed>; rel="next", <https://api.github.com/repos/path11/hexabat/issues?page=3&per_page=100&state=closed>; rel="last", <https://api.github.com/repos/path11/hexabat/issues?page=1&per_page=100&state=closed>; rel="first", <https://api.github.com/repos/path11/hexabat/issues?page=1&per_page=100&state=closed>; rel="prev"'
+  stub_request(:get, "https://api.github.com/repos/#{repository}/issues?page=2&per_page=100&state=closed").
+    to_return(:status => 200, :body => CLOSED_ISSUES_PAGE_2, :headers => {'Link' => link})
+
+  #closed issues page 3
+  link = '<https://api.github.com/repos/path11/hexabat/issues?page=1&per_page=100&state=closed>; rel="last", <https://api.github.com/repos/path11/hexabat/issues?page=1&per_page=100&state=closed>; rel="first", <https://api.github.com/repos/path11/hexabat/issues?page=2&per_page=100&state=closed>; rel="prev"'
+  stub_request(:get, "https://api.github.com/repos/#{repository}/issues?page=3&per_page=100&state=closed").
+    to_return(:status => 200, :body => CLOSED_ISSUES_PAGE_3, :headers => {'Link' => link})
+end
+
+Given /^the repository "(.*?)" doesn't exist$/ do |repository|
+  stub_request(:get, "https://api.github.com/repos/#{repository}/issues?page=1&per_page=100&state=closed").
+    to_return(:status => 404, :body => '{"message": "Not found"}', :headers => {})
+  stub_request(:get, "https://api.github.com/repos/#{repository}/issues?page=1&per_page=100&state=open").
+    to_return(:status => 404, :body => '{"message": "Not found"}', :headers => {})
+end
+
+When /^I set up an issue retrieved callback$/ do
+  setup_callbacks
+end
+
+When /^I setup the issue count known callback$/ do
+  setup_callbacks
+end
+
+When /^I import the "(.*?)" repository$/ do |repository|
+  EM.run do
+    EM.add_timer(TIMEOUT){ EM.stop }
+    Hexabat.import repository
+  end
+end
+
+When /^I setup the errback$/ do
+  setup_errback
+end
+
+Then /^the callback is called with the issue in that repository$/ do
+  @retrieved_issues.count.should be 1
+end
+
+Then /^the callback is called with the number of issues of the repository$/ do
+  @issue_count.should eq 401
+  @retrieved_issues.count.should eq @issue_count
+end
+
+Then /^the errback is called with the error message$/ do
+  @error_repository = 'path11/rails'
+  @error_status = '404'
+  @error_message = 'Not found'
+end

data/features/support/env.rb

@@ -0,0 +1,37 @@
+$LOAD_PATH.unshift File.join(File.dirname(__FILE__), '..', '..', 'lib')
+
+require 'hexabat'
+require 'webmock/cucumber'
+
+TIMEOUT = 2
+
+module CallbackSetup
+  def setup_callbacks
+    setup_issue_count_known
+    setup_issue_retrieved
+  end
+
+  def setup_errback
+    Hexabat.on_error do |repository, status, message|
+      @error_repository = repository
+      @error_status = status
+      @error_message = message
+    end
+  end
+
+  def setup_issue_retrieved
+    @retrieved_issues = []
+    Hexabat.on_issue_retrieved do |issue|
+      @retrieved_issues << issue
+      EM.stop if @retrieved_issues.count == @issue_count_known
+    end
+  end
+
+  def setup_issue_count_known
+    @issue_count = 0
+    Hexabat.on_issue_count_known { |count| @issue_count = count }
+  end
+end
+
+World(CallbackSetup)
+World(WebMock::API)

data/hexabat.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+require File.expand_path('../lib/hexabat/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = %w{path11}
+  gem.email         = %w{alberto@path11.com ecomba@path11.com javier@path11.com sebastian@path11.com}
+  gem.description   = %q{A Github issues importing tool}
+  gem.summary       = %q{Importing all the issues of a Github repository is a complex task: the Github API does not provide an easy way of doing it. Hexabat will help you with that. It will allow you to find out the total number of issues (counting both open and closed ones) and to perform an action with each one of them.}
+  gem.homepage      = ""
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "hexabat"
+  gem.require_paths = ["lib"]
+  gem.version       = Hexabat::VERSION
+
+  gem.add_runtime_dependency     'eventmachine', '~> 1.0.0'
+  gem.add_runtime_dependency     'em-http-request'
+  gem.add_runtime_dependency     'yajl-ruby'
+
+  gem.add_development_dependency 'cucumber'
+  gem.add_development_dependency 'rake'
+  gem.add_development_dependency 'rspec'
+  gem.add_development_dependency 'webmock'
+end

data/lib/hexabat/client.rb

@@ -0,0 +1,68 @@
+require 'eventmachine'
+
+require_relative 'importer'
+require_relative 'issue_count'
+require_relative 'page_request_builder'
+require_relative 'page_response_processor'
+
+module Hexabat
+  class Client
+    attr_reader :repository, :params, :callbacks
+
+    def initialize(repository, params = {})
+      @repository = repository
+      @params = params
+      @callbacks = {
+        issue_retrieved:   ->(issue){},
+        issue_count_known: ->(issue_count){},
+        error: ->(repository, status, message) { STDERR.puts "#{repository} #{status} #{message}" }
+      }
+    end
+
+    def import
+      if EM.reactor_running?
+        start_importing
+      else
+        EM.run &method(:start_importing)
+      end
+    end
+
+    def on(event_callback)
+      event = event_callback.keys.first
+      raise_unknown_event_error event unless known? event
+      callbacks.merge! event_callback
+    end
+
+    def known_events
+      callbacks.keys
+    end
+
+    private
+
+    def start_importing
+      Importer.new(issue_count, request_creator).import
+    end
+
+    def request_creator
+      PageRequestBuilder.for repository, response_processor, params
+    end
+
+    def issue_count
+      IssueCount.new(&callbacks[:issue_count_known])
+    end
+
+    def response_processor
+      PageResponseProcessor.new repository, callbacks[:issue_retrieved], callbacks[:error]
+    end
+
+    def known? event
+      known_events.include? event
+    end
+
+    def raise_unknown_event_error event
+      known = known_events.join(', ')
+      raise ArgumentError.new "Unknown #{event} event. Events: #{known}"
+    end
+
+  end
+end

data/lib/hexabat/importer.rb

@@ -0,0 +1,45 @@
+module Hexabat
+  class Importer
+    def initialize(issue_counter, request_creator)
+      @issue_counter = issue_counter
+      @request_creator = request_creator
+    end
+
+    def import
+      retrieve_first_page_of_issues :open
+      retrieve_first_page_of_issues :closed
+    end
+
+    def first_page_retrieved(state)
+      ->(page_range, issue_count) do
+        notify_counted_page :first, state, page_range, issue_count
+        retrieve_last_page_of_issues state, page_range.last if page_range.multiple_pages?
+      end
+    end
+
+    def last_page_retrieved(state)
+      ->(page_range, issue_count) do
+        notify_counted_page :last, state, page_range, issue_count
+        retrieve_pages_of_issues state, page_range.middle
+      end
+    end
+
+    private
+
+    def retrieve_first_page_of_issues(state)
+      @request_creator.for page: 1, state: state.to_s, &first_page_retrieved(state)
+    end
+
+    def retrieve_last_page_of_issues(state, page)
+      @request_creator.for page: page, state: state.to_s, &last_page_retrieved(state)
+    end
+
+    def retrieve_pages_of_issues(state, pages)
+      pages.each { |page| @request_creator.for page: page, state: state.to_s }
+    end
+
+    def notify_counted_page(first_or_last, state, page_range, issue_count)
+      @issue_counter.counted first_or_last, state, page_range, issue_count
+    end
+  end
+end

data/lib/hexabat/issue_count.rb

@@ -0,0 +1,49 @@
+module Hexabat
+  class IssueCount
+    ISSUES_PER_PAGE = 100
+
+    attr_reader :page_ranges, :issue_counts
+
+    def initialize(&issue_count_known)
+      @count_known  = issue_count_known
+      @page_ranges  = { open: nil, closed: nil }
+      @issue_counts = { open: {first: nil, last: nil}, closed: {first: nil, last: nil}}
+    end
+
+    def counted(page, state, page_range, issue_count)
+      check_done_counting do
+        @page_ranges[state] = page_range
+        @issue_counts[state][page] = issue_count
+        @issue_counts[state][:last] = 0 unless page_range.multiple_pages?
+      end
+    end
+
+    private
+
+    def check_done_counting
+      yield
+      @count_known.call(count) if all_done?
+    end
+
+    def all_done?
+      done_counting_issues? :open and done_counting_issues? :closed
+    end
+
+    def count
+      issue_count(:open) + issue_count(:closed)
+    end
+
+    def done_counting_issues?(state)
+      not @page_ranges[state].nil? and
+      not @issue_counts[state][:first].nil? and
+      not @issue_counts[state][:last].nil?
+    end
+
+    def issue_count(state)
+      @issue_counts[state][:first] +
+      @issue_counts[state][:last] +
+      @page_ranges[state].middle_page_count * ISSUES_PER_PAGE
+    end
+
+  end
+end

data/lib/hexabat/page_range.rb

@@ -0,0 +1,126 @@
+require 'delegate'
+
+module Hexabat
+  module PageRange
+    def self.from(headers)
+      return SinglePageRange.new(1) unless headers.has_key? 'LINK'
+      return MultiplePageRange.new(1, LinkHeader.new(headers['LINK']).last)
+    end
+
+    class LinkHeader
+      PREV = /<https:\/\/\S+\?page=(?'prev_page'\d+)[\S]+>; rel="prev"/
+      LAST = /<https:\/\/\S+\?page=(?'last_page'\d+)[\S]+>; rel="last"/
+
+      def initialize(header)
+        @header = header
+      end
+
+      def last
+        return prev.succ if last_missing? or last_incorrect?
+        return header_last
+      end
+
+      def prev
+        @prev ||= extract(PREV)
+      end
+
+      private
+
+      def header_last
+        @last ||= extract(LAST)
+      end
+
+      def extract(page_regexp)
+        @header.scan(page_regexp).flatten.map(&:to_i).first
+      end
+
+      def last_missing?
+        header_last.nil?
+      end
+
+      def last_incorrect?
+        # This check of the last page is needed because some times Github
+        # returns a LINK header with a wrong last page.
+        #
+        # This happens when you request the last page of a repository. When you
+        # do that the LINK header returns a link to the first page as the last
+        # page.
+        #
+        # See: https://gist.github.com/3754973
+        not prev.nil? and header_last < prev
+      end
+    end
+  end
+
+  class MultiplePageRange < DelegateClass(Range)
+    def initialize(first, last)
+      super(first..last)
+    end
+
+    def page_count
+      last - first + 1
+    end
+
+    def middle
+      if page_count > 3
+        MultiplePageRange.new(first.succ, last.pred)
+      elsif page_count == 3
+        SinglePageRange.new(first.succ)
+      else
+        EmptyPageRange.new
+      end
+    end
+
+    def middle_page_count
+      middle.page_count
+    end
+
+    def multiple_pages?
+      true
+    end
+  end
+
+  class SinglePageRange < DelegateClass(Range)
+    def initialize(page)
+      super page..page
+    end
+
+    def page_count
+      1
+    end
+
+    def middle
+      EmptyPageRange.new
+    end
+
+    def middle_page_count
+      middle.page_count
+    end
+
+    def multiple_pages?
+      false
+    end
+  end
+
+  class EmptyPageRange < DelegateClass(Range)
+    def initialize
+      super 0...0
+    end
+
+    def page_count
+      0
+    end
+
+    def middle
+      EmptyPageRange.new
+    end
+
+    def middle_page_count
+      middle.page_count
+    end
+
+    def multiple_pages?
+      false
+    end
+  end
+end

data/lib/hexabat/page_request_builder.rb

@@ -0,0 +1,76 @@
+require 'em-http-request'
+require 'em-http/middleware/json_response'
+require 'hexabat/page_range'
+
+module Hexabat
+  module PageRequestBuilder
+
+    def self.for(repository, response_processor, params = {})
+      if params.has_key? :token
+        TokenAuthorized.new(repository, params[:token], response_processor)
+      elsif params.has_key? :oauth
+        raise 'OAuth not supported yet'
+      else
+        Unauthorized.new(repository, response_processor)
+      end
+    end
+
+    class Base
+      MAX_PAGE_SIZE = 100
+
+      def initialize(repository, response_processor)
+        @repository = repository
+        @response_processor = response_processor
+        EM::HttpRequest.use(EM::Middleware::JSONResponse)
+      end
+
+      def for(params, &page_callback)
+        build_request(query_from params).callback &page_retrieved(page_callback)
+      end
+
+      def page_retrieved(page_callback = nil)
+        ->(http) { @response_processor.process(http, &page_callback) }
+      end
+
+      def endpoint
+        "https://api.github.com/repos/#{@repository}/issues"
+      end
+
+      private
+
+      def build_request(query)
+        raise "PageRequestBuilder::Base does not know how to build requests. Use one if it's subclasses"
+      end
+
+      def query_from(params)
+        params.merge per_page: MAX_PAGE_SIZE
+      end
+    end
+
+    class Unauthorized < Base
+      def build_request(query)
+        EM::HttpRequest.new(endpoint).get query: query
+      end
+
+      private :build_request
+    end
+
+    class TokenAuthorized < Base
+      def initialize(repository, token, response_processor)
+       @token = token
+       super(repository, response_processor)
+      end
+
+      def build_request(query)
+        EM::HttpRequest.new(endpoint).get query: query, head: headers
+      end
+
+      def headers
+        { 'Authorization' => "token #{@token}" }
+      end
+
+      private :build_request, :headers
+    end
+
+  end
+end

data/lib/hexabat/page_response_processor.rb

@@ -0,0 +1,35 @@
+require 'eventmachine'
+require 'hexabat/page_range'
+
+module Hexabat
+  class PageResponseProcessor
+    def initialize(repository, issue_callback, errback)
+      @repository = repository
+      @issue_callback = issue_callback
+      @errback = errback
+    end
+
+    def process(http, &page_callback)
+      check_for_errors(http)
+      process_response(http.response_header, http.response, page_callback)
+    end
+
+    private
+
+    def check_for_errors(http)
+      status = http.response_header.status
+      @errback.call @repository, status, http.response['message'] if status > 200
+    end
+
+    def process_response(headers, issues, page_callback)
+      page_callback.call PageRange.from(headers), issues.count unless page_callback.nil?
+      notify_issue_retrieved issues
+    end
+
+    def notify_issue_retrieved(issues)
+      issues.each do |issue|
+        EM.next_tick { @issue_callback.call issue }
+      end
+    end
+  end
+end

data/lib/hexabat/version.rb

@@ -0,0 +1,3 @@
+module Hexabat
+  VERSION = "0.0.1"
+end

data/lib/hexabat.rb

@@ -0,0 +1,26 @@
+require 'hexabat/client'
+require 'hexabat/importer'
+require 'hexabat/version'
+
+module Hexabat
+  def self.import(repository, params = {})
+    Client.new(repository, params).tap do |hexabat|
+      hexabat.on issue_retrieved:   @issue_retrieved
+      hexabat.on issue_count_known: @issue_count_known
+      hexabat.on error:             @errback
+      hexabat.import
+    end
+  end
+
+  def self.on_issue_retrieved(&callback)
+    @issue_retrieved = callback
+  end
+
+  def self.on_issue_count_known(&callback)
+    @issue_count_known = callback
+  end
+
+  def self.on_error(&callback)
+    @errback = callback
+  end
+end

data/spec/hexabat/client_spec.rb

@@ -0,0 +1,45 @@
+require 'hexabat/client'
+
+describe Hexabat::Client do
+  subject          { described_class.new(repository) }
+  let(:repository) { 'path11/hexabat' }
+  let(:a_callback) { lambda {} }
+
+  it 'can be set with the issue retrieved callback' do
+    subject.on issue_retrieved: a_callback
+    subject.callbacks[:issue_retrieved].should be a_callback
+  end
+
+  it 'can be set with the issue count retrieved callback' do
+    subject.on issue_count_known: a_callback
+    subject.callbacks[:issue_count_known].should be a_callback
+  end
+
+  it 'raises an error if a callback for an unknown event is set' do
+    expect { subject.on unknown_event: a_callback}.to raise_error ArgumentError
+  end
+
+  it 'starts Event Machine if it is not running' do
+    EM.stub(:reactor_running?).and_return(false)
+    EM.should_receive(:run)
+    subject.stub(:start_importing)
+    subject.import
+  end
+
+  it 'does not start Event Machine if it is already running' do
+    EM.stub(:reactor_running?).and_return(true)
+    EM.should_not_receive(:run)
+    subject.stub(:start_importing)
+    subject.import
+  end
+
+  it 'imports all the issues from the repository' do
+    importer = mock(:importer)
+    EM.stub(:run).and_yield
+    Hexabat::Importer.stub(:new).and_return(importer)
+    importer.should_receive(:import)
+    subject.on issue_retrieved: :callback1
+    subject.on issue_count_known: :callback2
+    subject.import
+  end
+end