ghtorrent 0.8 → 0.8.1

data/lib/ghtorrent/logging.rb

@@ -38,8 +38,9 @@ module GHTorrent
       end
     end
 
+    # Default logger
     def logger
-      raise Exception.new("Unimplemented")
+      @logger ||= Logger.new(STDOUT)
     end
   end
 end

data/lib/ghtorrent/migrations/015_fix_table_issue_labels.rb

@@ -6,12 +6,24 @@ Sequel.migration do
   up do
     puts "Fixing table issue_labels"
 
-    alter_table :issue_labels do
-      drop_column :repo_id
-      drop_column :ext_ref_id
-      add_foreign_key :issue_id, :issues
-      add_primary_key ([:issue_id, :label_id])
+    #alter_table :issue_labels do
+    #  drop_constraint 'issue_labels_ibfk_1'
+    #  drop_constraint 'issue_labels_ibfk_2'
+    #  drop_column :repo_id
+    #  drop_column :ext_ref_id
+    #  drop_column :label_id
+      #add_foreign_key :label_id, :repo_labels
+      #add_foreign_key :issue_id, :issues
+      #add_primary_key ([:issue_id, :label_id])
+    #end
+
+    drop_table :issue_labels
+    create_table :issue_labels do
+      foreign_key :label_id, :repo_labels
+      foreign_key :issue_id, :issues
+      primary_key [:issue_id, :label_id]
     end
+    
   end
 
   down do

data/lib/ghtorrent/migrations/016_add_actor_pull_request_history.rb

@@ -7,7 +7,7 @@ Sequel.migration do
     puts "Adding column user_id to table pull_rq"
 
     alter_table :pull_request_history do
-      add_foreign_key :actor_id, :user
+      add_foreign_key :actor_id, :users
     end
 
     puts 'Remember to run the fixes/update_pull_request_history_actor.rb

data/lib/ghtorrent/retriever.rb

@@ -216,16 +216,11 @@ module GHTorrent
     end
 
     # Retrieve all comments for a single commit
-    def retrieve_commit_comments(user, repo, sha)
-      retrieved_comments = paged_api_request(ghurl "repos/#{user}/#{repo}/commits/#{sha}/comments")
+    def retrieve_commit_comments(owner, repo, sha)
+      retrieved_comments = paged_api_request(ghurl "repos/#{owner}/#{repo}/commits/#{sha}/comments")
 
       retrieved_comments.each { |x|
-        x['repo'] = repo
-        x['user'] = user
-        x['commit_id'] = sha
-
-        if persister.find(:commit_comments, {'repo' => repo,
-                                              'user' => user,
+        if persister.find(:commit_comments, { 'commit_id' => x['commit_id'],
                                               'id' => x['id']}).empty?
           persister.store(:commit_comments, x)
         end
@@ -234,25 +229,21 @@ module GHTorrent
     end
 
     # Retrieve a single comment
-    def retrieve_commit_comment(user, repo, id)
+    def retrieve_commit_comment(owner, repo, sha, id)
 
-      comment = persister.find(:commit_comments, {'repo' => repo,
-                                                   'user' => user,
-                                                   'id' => id}).first
+      comment = persister.find(:commit_comments, {'commit_id' => sha,
+                                                  'id' => id}).first
       if comment.nil?
-        r = api_request(ghurl "repos/#{user}/#{repo}/comments/#{id}")
+        r = api_request(ghurl "repos/#{owner}/#{repo}/comments/#{id}")
 
         if r.empty?
           debug "Retriever: Commit comment #{id} deleted"
           return
         end
 
-        r['repo'] = repo
-        r['user'] = user
         persister.store(:commit_comments, r)
         info "Retriever: Added commit comment #{r['commit_id']} -> #{r['id']}"
-        persister.find(:commit_comments, {'repo' => repo, 'user' => user,
-                                          'id' => id}).first
+        persister.find(:commit_comments, {'commit_id' => sha, 'id' => id}).first
       else
         debug "Retriever: Commit comment #{comment['commit_id']} -> #{comment['id']} exists"
         comment

data/lib/ghtorrent/settings.rb

@@ -34,7 +34,9 @@ module GHTorrent
 
         :respect_api_ratelimit => 'mirror.respect_api_ratelimit',
 
-        :attach_ip => 'mirror.attach_ip'
+        :attach_ip => 'mirror.attach_ip',
+
+        :rescue_loops => 'mirror.rescue_loops'
     }
 
     DEFAULTS = {
@@ -63,7 +65,9 @@ module GHTorrent
 
         :respect_api_ratelimit => 'true',
 
-        :attach_ip => '0.0.0.0'
+        :attach_ip => '0.0.0.0',
+
+        :rescue_loops => 'true'
     }
 
     def config(key, use_default = true)
@@ -87,9 +91,9 @@ module GHTorrent
       more_keys.each {|k,v| CONFIGKEYS[k] = v}
     end
 
-    def merge_config_values(values)
-      values.reduce(settings) {|acc, k|
-        acc.merge_recursive write_value(settings, CONFIGKEYS[k[0]], k[1])
+    def merge_config_values(config, values)
+      values.reduce(config) {|acc, k|
+        acc.merge_recursive write_value(config, CONFIGKEYS[k[0]], k[1])
       }
     end
 

data/lib/ghtorrent/transacted_ghtorrent.rb

@@ -0,0 +1,91 @@
+require 'ghtorrent/ghtorrent'
+
+
+# A version of the GHTorrent class that creates a transaction per processed
+# item
+class TransactedGhtorrent < GHTorrent::Mirror
+
+  def ensure_commit(repo, sha, user, comments = true)
+    check_transaction do
+      super(repo, sha, user, comments)
+    end
+  end
+
+  def ensure_commit_comment(owner, repo, sha, comment_id)
+    check_transaction do
+      super(owner, repo, sha, comment_id)
+    end
+  end
+
+  def ensure_fork(owner, repo, fork_id)
+    check_transaction do
+      super(owner, repo, fork_id)
+    end
+  end
+
+  def ensure_pull_request(owner, repo, pullreq_id,
+      comments = true, commits = true, history = true,
+      state = nil, actor = nil, created_at = nil)
+    check_transaction do
+      super(owner, repo, pullreq_id, comments, commits, history, state, actor, created_at)
+    end
+  end
+
+  def ensure_pullreq_comment(owner, repo, pullreq_id, comment_id)
+    check_transaction do
+      super(owner, repo, pullreq_id, comment_id)
+    end
+  end
+
+  def ensure_issue(owner, repo, issue_id, events = true, comments = true, labels = true)
+    check_transaction do
+      super(owner, repo, issue_id, events, comments, labels)
+    end
+  end
+
+  def ensure_issue_event(owner, repo, issue_id, event_id)
+    check_transaction do
+      super(owner, repo, issue_id, event_id)
+    end
+  end
+
+  def ensure_issue_comment(owner, repo, issue_id, comment_id, pull_req_id = nil)
+    check_transaction do
+      super(owner, repo, issue_id, comment_id, pull_req_id)
+    end
+  end
+
+  def ensure_issue_label(owner, repo, issue_id, name)
+    check_transaction do
+      super(owner, repo, issue_id, name)
+    end
+  end
+
+  def ensure_project_member(owner, repo, new_member, date_added)
+    check_transaction do
+      super(owner, repo, new_member, date_added)
+    end
+  end
+
+  def ensure_watcher(owner, repo, watcher, date_added = nil)
+    check_transaction do
+      super(owner, repo, watcher, date_added)
+    end
+  end
+
+  def ensure_repo_label(owner, repo, name)
+    check_transaction do
+      super(owner, repo, name)
+    end
+  end
+
+  def check_transaction(&block)
+    if @db.in_transaction?
+      yield block
+    else
+      transaction do
+        yield block
+      end
+    end
+  end
+end

data/lib/version.rb

@@ -1,5 +1,5 @@
 module GHTorrent
 
-  VERSION = '0.8'
+  VERSION = '0.8.1'
 
 end

data/spec/api_client_spec.rb

@@ -0,0 +1,42 @@
+require 'spec_helper'
+
+include GHTorrent::Settings
+include GHTorrent::APIClient
+
+describe 'The API client' do
+
+  def stub_config(remaining, reset)
+    stub_request(:get, "https://#{config(:github_username)}:#{config(:github_passwd)}@api.github.com/events").
+        with(:headers => {'Accept'=>'*/*', 'User-Agent'=>'ghtorrent'}).
+        to_return(:status => 200, :body => (1..30).to_a,
+                  :headers => {'x-ratelimit-remaining' => remaining,
+                               'x-ratelimit-reset' => reset})
+  end
+
+  it 'call /events should return 30 results' do
+    stub_config(1000, 1000)
+    r = api_request 'https://api.github.com/events', false
+    expect(r.size).to eq(30)
+  end
+
+  it 'should resume from sleep' do
+    time = Time.now.to_i
+    stub_config(2, Time.now.to_i + 1)
+    api_request 'https://api.github.com/events', false
+
+
+    stub_config(1000, 1000)
+    api_request 'https://api.github.com/events', false
+    expect(Time.now.to_i - time).to be_within(0.3).of(3)
+  end
+
+  it 'should sleep for 7 seconds' do
+    stub_config(2, Time.now.to_i + 5)
+
+    time = Time.now.to_i
+    api_request 'https://api.github.com/events', false
+    expect(Time.now.to_i - time).to be_within(0.3).of(7)
+  end
+
+end
+

data/spec/spec_helper.rb

@@ -0,0 +1,21 @@
+require 'rspec'
+require 'webmock/rspec'
+
+require 'ghtorrent'
+
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+
+RSpec.configure do |config|
+  config.expect_with :rspec do |c|
+    c.syntax = :expect
+  end
+end
+
+def settings
+  YAML::load_file('config.yaml')
+end
+
+def logger
+  Logger.new(STDERR)
+end

metadata

@@ -1,8 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ghtorrent
 version: !ruby/object:Gem::Version
-  version: '0.8'
-  prerelease: 
+  version: 0.8.1
 platform: ruby
 authors:
 - Georgios Gousios
@@ -10,60 +9,53 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-23 00:00:00.000000000 Z
+date: 2013-12-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: amqp
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.1.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.0.0
+        version: 1.1.0
 - !ruby/object:Gem::Dependency
   name: mongo
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.8.0
+        version: 1.9.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.8.0
+        version: 1.9.0
 - !ruby/object:Gem::Dependency
   name: bson_ext
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.8.0
+        version: 1.9.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.8.0
+        version: 1.9.0
 - !ruby/object:Gem::Dependency
   name: trollop
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -71,7 +63,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -79,37 +70,48 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: sequel
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '3.47'
+        version: 4.5.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '3.47'
+        version: 4.5.0
 - !ruby/object:Gem::Dependency
-  name: daemons
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.1.0
-  type: :runtime
+        version: 2.14.0
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.1.0
-description: ! "A library and a collection of associated programs\n                    to
-  mirror and process Github data"
+        version: 2.14.0
+- !ruby/object:Gem::Dependency
+  name: webmock
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.16'
+description: |-
+  A library and a collection of associated programs
+                      to mirror and process Github data
 email: gousiosg@gmail.com
 executables:
 - ght-data-retrieval
@@ -123,6 +125,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- lib/ghtorrent.rb
 - lib/ghtorrent/adapters/base_adapter.rb
 - lib/ghtorrent/adapters/mongo_persister.rb
 - lib/ghtorrent/adapters/noop_persister.rb
@@ -134,7 +137,9 @@ files:
 - lib/ghtorrent/commands/ght_get_more_commits.rb
 - lib/ghtorrent/commands/ght_load.rb
 - lib/ghtorrent/commands/ght_mirror_events.rb
+- lib/ghtorrent/commands/ght_retrieve_dependents.rb
 - lib/ghtorrent/commands/ght_retrieve_repo.rb
+- lib/ghtorrent/commands/ght_retrieve_repos.rb
 - lib/ghtorrent/commands/ght_retrieve_user.rb
 - lib/ghtorrent/commands/ght_rm_dupl.rb
 - lib/ghtorrent/gh_torrent_exception.rb
@@ -162,60 +167,49 @@ files:
 - lib/ghtorrent/retriever.rb
 - lib/ghtorrent/settings.rb
 - lib/ghtorrent/time.rb
+- lib/ghtorrent/transacted_ghtorrent.rb
 - lib/ghtorrent/utils.rb
-- lib/ghtorrent.rb
 - lib/version.rb
 - bin/ght-data-retrieval
 - bin/ght-get-more-commits
 - bin/ght-load
 - bin/ght-mirror-events
 - bin/ght-process-event
+- bin/ght-retrieve-dependents
 - bin/ght-retrieve-repo
+- bin/ght-retrieve-repos
 - bin/ght-retrieve-user
 - bin/ght-rm-dupl
 - CHANGELOG
 - Gemfile
 - Gemfile.lock
 - LICENSE
-- Rakefile
 - README.md
+- Rakefile
+- spec/api_client_spec.rb
+- spec/spec_helper.rb
 homepage: https://github.com/gousiosg/github-mirror
 licenses: []
-post_install_message: !binary |-
-  WxtbMzJtVmVyc2lvbiAwLjgbWzBtXSBSZXRyaWV2ZSBhbmQgcHJvY2VzcyBp
-  c3N1ZSBsYWJlbHMKWxtbMzJtVmVyc2lvbiAwLjgbWzBtXSBSZXRyaXZlIGFu
-  ZCBwcm9jZXNzIGFjdG9ycyBmb3IgcHVsbCByZXF1ZXN0IGV2ZW50cwpbG1sz
-  Mm1WZXJzaW9uIDAuOBtbMG1dIFJldHJpZXZlIHB1bGxyZXEgY29tbWVudHMg
-  Zm9yIHByb2plY3RzIHdpdGggbm8gaXNzdWV0cmFja2VyClsbWzMybVZlcnNp
-  b24gMC44G1swbV0gQmV0dGVyIGhldXJpc3RpY3MgZm9yIGF2b2lkaW5nIGR1
-  bHBpY2F0ZSBlbnRyaWVzIGluIHB1bGwgcmVxdWVzdCBoaXN0b3JpZXMKWxtb
-  MzJtVmVyc2lvbiAwLjgbWzBtXSBUaGUgZXZlbnQgbG9hZGVyIG5vdyBsb2Fk
-  cyBldmVudHMgYnkgSURzIHRvIHJlZHVjZSBwcmVzc3VyZSB0byB0aGUgcXVl
-  dWUKWxtbMzJtVmVyc2lvbiAwLjgbWzBtXSBDb21wb3VuZCBpbmRleGVzIGlu
-  IE1vbmdvREIgYnkgZGVmYXVsdApbG1szMm1WZXJzaW9uIDAuOBtbMG1dIFgt
-  UmF0ZUxpbWl0LVJlc2V0IGhlYWRlciBzdXBwb3J0ClsbWzMybVZlcnNpb24g
-  MC44G1swbV0gUmVtb3ZlIGxvdHMgb2YgZGVhZCBjb2RlLCBnZW5lcmFsIGNs
-  ZWFudXBzCg==
+metadata: {}
+post_install_message: 
 rdoc_options:
 - --charset=UTF-8
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.25
+rubygems_version: 2.0.3
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Mirror and process Github data
 test_files: []