ghtorrent 0.5 → 0.6

data/lib/ghtorrent/hash.rb

@@ -4,7 +4,7 @@ class Hash
       if x.respond_to?(:merge_recursive) && y.is_a?(Hash)
         x.merge_recursive(y)
       else
-        if overwrite then y else [*x,*y] end
+        if overwrite then y else [x, y].flatten.uniq end
       end
     end
   end

data/lib/ghtorrent/migrations/011_add_issues.rb

@@ -0,0 +1,74 @@
+require 'sequel'
+
+require 'ghtorrent/migrations/mysql_defaults'
+
+Sequel.migration do
+  up do
+
+    puts("Adding table issues")
+    create_table :issues do
+      primary_key :id
+      foreign_key :repo_id, :projects
+      foreign_key :reporter_id, :users, :null => true
+      foreign_key :assignee_id, :users, :null => true
+      Long :issue_id, :null =>  false
+      TrueClass :pull_request, :null => false
+      foreign_key :pull_request_id, :pull_requests, :null => true
+      DateTime :created_at, :null => false, :default=>Sequel::CURRENT_TIMESTAMP
+      String :ext_ref_id, :null => false, :size => 24, :default => "0"
+    end
+
+    puts("Adding issue history")
+    create_table :issue_events do
+      Long :event_id, :null => false
+      foreign_key :issue_id, :issues, :null => false
+      foreign_key :actor_id, :users, :null => false
+      String :action, :null => false
+      String :action_specific, :null => true, :size => 50
+      DateTime :created_at, :null => false, :default=>Sequel::CURRENT_TIMESTAMP
+      String :ext_ref_id, :null => false, :size => 24, :default => "0"
+      check(:action=>%w[closed reopened subscribed merged referenced mentioned assigned])
+      primary_key(:event_id, :issue_id)
+    end
+
+    puts("Adding table issue comments")
+    create_table :issue_comments do
+      foreign_key :issue_id, :issues, :null => false
+      foreign_key :user_id, :users, :null => false
+      Long :comment_id, :null =>  false
+      DateTime :created_at, :null => false, :default=>Sequel::CURRENT_TIMESTAMP
+      String :ext_ref_id, :null => false, :size => 24, :default => "0"
+    end
+
+    puts("Adding table repo labels")
+    create_table :repo_labels do
+      primary_key :id
+      foreign_key :repo_id, :projects
+      String :name, :size => 24, :null => false
+      String :ext_ref_id, :null => false, :size => 24, :default => "0"
+    end
+
+    puts("Adding table issue labels")
+    create_table :issue_labels do
+      foreign_key :label_id, :repo_labels
+      foreign_key :repo_id, :projects
+      String :ext_ref_id, :null => false, :size => 24, :default => "0"
+    end
+
+    puts("Adding table repo milestones")
+    create_table :repo_milestones do
+      primary_key :id
+      foreign_key :repo_id, :projects
+      String :name, :size => 24, :null => false
+      String :ext_ref_id, :null => false, :size => 24, :default => "0"
+    end
+  end
+
+  down do
+    drop_table :repo_milestones
+    drop_table :repo_labels
+    drop_table :issue_comments
+    drop_table :issue_events
+    drop_table :issues
+  end
+end

data/lib/ghtorrent/retriever.rb

@@ -13,11 +13,11 @@ module GHTorrent
     include GHTorrent::APIClient
 
     def ext_uniq
-      raise Exception("Unimplemented")
+      raise Exception.new("Unimplemented")
     end
 
     def persister
-      raise Exception("Unimplemented")
+      raise Exception.new("Unimplemented")
     end
 
     def retrieve_user_byusername(user)
@@ -120,11 +120,11 @@ module GHTorrent
 
     # Retrieve up to 30 * +:mirror_commit_pages_new_repo+ commits
     # starting from the provided +sha+
-    def retrieve_commits(repo, sha, user)
-      last_sha = if sha.nil? then "master" else sha end
+    def retrieve_commits(repo, sha, user, num_pages = config(:mirror_commit_pages_new_repo))
+      last_sha = if sha == "head" then "master" else sha end
 
-      url = ghurl "repos/#{user}/#{repo}/commits?last_sha=#{last_sha}"
-      commits = paged_api_request(url, config(:mirror_commit_pages_new_repo))
+      url = ghurl "repos/#{user}/#{repo}/commits?sha=#{last_sha}"
+      commits = paged_api_request(url, num_pages)
 
       commits.map do |c|
         retrieve_commit(repo, c['sha'], user)
@@ -251,7 +251,7 @@ module GHTorrent
     # Retrieve all watchers for a repository
     def retrieve_watchers(user, repo)
       repo_bound_items(user, repo, :watchers,
-                       "repos/#{user}/#{repo}/watchers",
+                       "repos/#{user}/#{repo}/stargazers",
                        {'repo' => repo, 'owner' => user},
                        'login')
     end
@@ -259,7 +259,7 @@ module GHTorrent
     # Retrieve a single watcher for a repositry
     def retrieve_watcher(user, repo, watcher)
       repo_bound_item(user, repo, watcher, :watchers,
-                      "repos/#{user}/#{repo}/watchers",
+                      "repos/#{user}/#{repo}/stargazers",
                       {'repo' => repo, 'owner' => user},
                       'login')
     end
@@ -371,17 +371,69 @@ module GHTorrent
     end
 
     def retrieve_issues(user, repo)
+      open = "repos/#{user}/#{repo}/issues"
+      closed = "repos/#{user}/#{repo}/issues?state=closed"
       repo_bound_items(user, repo, :issues,
-                       "repos/#{user}/#{repo}/issues",
+                       [open, closed],
                        {'repo' => repo, 'owner' => user},
-                       'id')
+                       'number')
     end
 
     def retrieve_issue(user, repo, issue_id)
+      open = "repos/#{user}/#{repo}/issues"
+      closed = "repos/#{user}/#{repo}/issues?state=closed"
       repo_bound_item(user, repo, issue_id, :issues,
-                      "repos/#{user}/#{repo}/issues/#{issue_id}",
+                      [open, closed],
                       {'repo' => repo, 'owner' => user},
-                      'id')
+                      'number')
+    end
+
+    def retrieve_issue_events(owner, repo, issue_id)
+      url = ghurl "repos/#{owner}/#{repo}/issues/#{issue_id}/events"
+      retrieved_events = paged_api_request url
+
+      retrieved_events.each { |x|
+        x['owner'] = owner
+        x['repo'] = repo
+        x['issue_id'] = issue_id
+
+        if persister.find(:issue_events, {'owner' => owner,
+                                          'repo' => repo,
+                                          'issue_id' => issue_id,
+                                          'id' => x['id']}).empty?
+          info "Retriever: Added issue event #{owner}/#{repo} #{issue_id}->#{x['id']}"
+          persister.store(:issue_events, x)
+        end
+      }
+      persister.find(:issue_events, {'owner' => owner, 'repo' => repo,
+                                     'issue_id' => issue_id})
+    end
+
+    def retrieve_issue_event(owner, repo, issue_id, event_id)
+      event = persister.find(:issue_events, {'repo' => repo,
+                                             'owner' => owner,
+                                             'issue_id' => issue_id,
+                                             'id' => event_id}).first
+      if event.nil?
+        r = api_request(ghurl "repos/#{owner}/#{repo}/issues/events/#{event_id}")
+
+        if r.empty?
+          warn "Retriever: Issue event #{owner}/#{repo} #{issue_id}->#{event_id} deleted"
+          return
+        end
+
+        r['repo'] = repo
+        r['owner'] = owner
+        r['issue_id'] = issue_id
+        persister.store(:issue_events, r)
+        info "Retriever: Added issue event #{owner}/#{repo} #{issue_id}->#{event_id}"
+        persister.find(:issue_events, {'repo' => repo, 'owner' => owner,
+                                       'issue_id' => issue_id,
+                                       'id' => event_id}).first
+      else
+        debug "Retriever: Issue event #{owner}/#{repo} #{issue_id}->#{event_id} exists"
+        event
+      end
     end
 
     def retrieve_issue_comments(owner, repo, issue_id)
@@ -410,16 +462,16 @@ module GHTorrent
                                                  'issue_id' => issue_id,
                                                  'id' => comment_id}).first
       if comment.nil?
-        r = api_request(ghurl "repos/#{owner}/#{repo}/issues/#{issue_id}/comments/#{comment_id}")
+        r = api_request(ghurl "repos/#{owner}/#{repo}/issues/comments/#{comment_id}")
 
         if r.empty?
-          debug "Retriever: Issue comment #{owner}/#{repo} #{issue_id}->#{comment_id} deleted"
+          warn "Retriever: Issue comment #{owner}/#{repo} #{issue_id}->#{comment_id} deleted"
           return
         end
 
         r['repo'] = repo
         r['owner'] = owner
-        x['issue_id'] = issue_id
+        r['issue_id'] = issue_id
         persister.store(:issue_comments, r)
         info "Retriever: Added issue comment #{owner}/#{repo} #{issue_id}->#{comment_id}"
         persister.find(:issue_comments, {'repo' => repo, 'owner' => owner,
@@ -427,7 +479,7 @@ module GHTorrent
                                          'id' => comment_id}).first
         r
       else
-        debug "Retriever: Commit comment #{owner}/#{repo} #{issue_id}->#{comment_id} exists"
+        debug "Retriever: Issue comment #{owner}/#{repo} #{issue_id}->#{comment_id} exists"
         comment
       end
     end
@@ -437,6 +489,26 @@ module GHTorrent
       api_request "https://api.github.com/events"
     end
 
+    # Get all events for the specified repo
+    def get_repo_events(owner, repo)
+      url = ghurl("repos/#{owner}/#{repo}/events")
+      r = paged_api_request(url)
+
+      r.each do |e|
+        if get_event(e['id']).empty?
+          info "Retriever: Already got event #{owner}/#{repo} -> #{e['id']}"
+        else
+          @persister.store(:events, e)
+          info "Retriever: Added event #{owner}/#{repo} -> #{e['id']}"
+        end
+      end
+    end
+
+    # Get a specific event by +id+.
+    def get_event(id)
+      persister.find(:events, {'id' => id})
+    end
+
     private
 
     def repo_bound_items(user, repo, entity, urls, selector, descriminator,

data/lib/ghtorrent/settings.rb

@@ -30,6 +30,11 @@ module GHTorrent
         :cache_dir       => "mirror.cache_dir",
         :cache_stale_age => "mirror.cache_stale_age",
 
+        :github_username => "mirror.username",
+        :github_passwd => "mirror.passwd",
+
+        :respect_api_ratelimit => "mirror.respect_api_ratelimit",
+
         :attach_ip => "mirror.attach_ip"
     }
 
@@ -48,7 +53,7 @@ module GHTorrent
     end
 
     def settings
-      raise Exception("Unimplemented")
+      raise Exception.new("Unimplemented")
     end
 
   end

data/lib/version.rb

@@ -1,5 +1,5 @@
 module GHTorrent
 
-  VERSION = '0.5'
+  VERSION = '0.6'
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ghtorrent
 version: !ruby/object:Gem::Version
-  version: '0.5'
+  version: '0.6'
   prerelease: 
 platform: ruby
 authors:
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-07 00:00:00.000000000 Z
+date: 2012-11-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: amqp
@@ -148,6 +148,8 @@ executables:
 - ght-mirror-events
 - ght-load
 - ght-rm-dupl
+- ght-process-event
+- ght-get-more-commits
 extensions: []
 extra_rdoc_files: []
 files:
@@ -158,6 +160,12 @@ files:
 - lib/ghtorrent/bson_orderedhash.rb
 - lib/ghtorrent/cache.rb
 - lib/ghtorrent/command.rb
+- lib/ghtorrent/commands/ght_data_retrieval.rb
+- lib/ghtorrent/commands/ght_get_more_commits.rb
+- lib/ghtorrent/commands/ght_load.rb
+- lib/ghtorrent/commands/ght_mirror_events.rb
+- lib/ghtorrent/commands/ght_retrieve_repo.rb
+- lib/ghtorrent/commands/ght_rm_dupl.rb
 - lib/ghtorrent/gh_torrent_exception.rb
 - lib/ghtorrent/ghtorrent.rb
 - lib/ghtorrent/hash.rb
@@ -172,6 +180,7 @@ files:
 - lib/ghtorrent/migrations/008_add_project_unq.rb
 - lib/ghtorrent/migrations/009_add_project_commit.rb
 - lib/ghtorrent/migrations/010_add_forks.rb
+- lib/ghtorrent/migrations/011_add_issues.rb
 - lib/ghtorrent/migrations/mysql_defaults.rb
 - lib/ghtorrent/persister.rb
 - lib/ghtorrent/retriever.rb
@@ -181,9 +190,12 @@ files:
 - lib/ghtorrent.rb
 - lib/version.rb
 - bin/ght-data-retrieval
+- bin/ght-get-more-commits
 - bin/ght-load
 - bin/ght-mirror-events
 - bin/ght-periodic-dump
+- bin/ght-process-event
+- bin/ght-retrieve-repo
 - bin/ght-rm-dupl
 - bin/ght-torrent-index
 - CHANGELOG
@@ -196,30 +208,28 @@ files:
 homepage: https://github.com/gousiosg/github-mirror
 licenses: []
 post_install_message: !binary |-
-  WxtbMzJtVmVyc2lvbiAwLjUbWzBtXSBHZW5lcmljIG1ldGhvZHMgZm9yIHJl
-  dHJpZXZpbmcgaXRlbXMgdGhhdCBhcmUgYm91bmQgdG8gcmVwb3NpdG9yaWVz
-  ClsbWzMybVZlcnNpb24gMC41G1swbV0gUHJvY2Vzc2luZyBvZiBwdWxsIHJl
-  cXVlc3RzIHdpdGggY29tbWl0cywgY29tbWVudHMgYW5kIGhpc3RvcnkKWxtb
-  MzJtVmVyc2lvbiAwLjUbWzBtXSBQcm9jZXNzaW5nIG9mIHByb2plY3QgZm9y
-  a3MKWxtbMzJtVmVyc2lvbiAwLjUbWzBtXSBOZXcgdG9vbCAoZ2h0LWxvYWQp
-  IHRvIGZpbHRlciBhbmQgbG9hZCBldmVudHMgdG8gdGhlIHF1ZXVlClsbWzMy
-  bVZlcnNpb24gMC41G1swbV0gTmV3IHRvb2wgKGdodC1ybS1kdXBsKSB0byBk
-  ZWxldGUgZHVwbGljYXRlIGVudHJpZXMgZnJvbSBjb2xsZWN0aW9ucyAoZXZl
-  bnRzICYgY29tbWl0cyBzdXBwb3J0ZWQpClsbWzMybVZlcnNpb24gMC41G1sw
-  bV0gUHJvamVjdCB3aWRlIHJlcXVlc3RpbmcgcmVzdWx0IGNhY2hpbmcgZm9y
-  IG11bHRpLXBhZ2UgcmVxdWVzdHMKWxtbMzJtVmVyc2lvbiAwLjUbWzBtXSBC
-  ZXR0ZXIgbG9nZ2luZyBpbiB2YXJpb3VzIHBsYWNlcwpbG1szMm1WZXJzaW9u
-  IDAuNRtbMG1dIEJldHRlciBkZWZhdWx0cyBmb3IgTXlTUUwgKFVURjggKyBJ
-  bm5vREIgdGFibGVzKQpbG1szMm1WZXJzaW9uIDAuNRtbMG1dIENvbW1pdHMg
-  YXJlIG5vdyBzZXBlcmF0ZWQgZnJvbSBwcm9qZWN0cy4gUHJvamVjdCBmb3Jr
-  cyBjYW4gc2hhcmUgY29tbWl0cy4KWxtbMzJtVmVyc2lvbiAwLjUbWzBtXSBT
-  dXBwb3J0IGZvciBzZXR0aW5nIHRoZSBJUCBhZGRyZXNzIHRvIHVzZSBmb3Ig
-  cmV0cmlldmFsIG9uIG11bHRpLWhvbWVkClsbWzMybVZlcnNpb24gMC41G1sw
-  bV0gQ29tcGF0aWJpbGl0eSB3aXRoIFJ1YnkgMS45IChub3cgZGVmYXVsdCkg
-  YW5kIEpSdWJ5ClsbWzMybVZlcnNpb24gMC41G1swbV0gUHJvcGVyIG1vZHVs
-  YXJpemF0aW9uLCBmb2xsb3dpbmcgdGhlIGNha2UgZGVzaWduIHBhdHRlcm4K
-  WxtbMzJtVmVyc2lvbiAwLjUbWzBtXSBOZXZlciByZXRyaWV2ZSBhcnJheXMg
-  b2YgcmVzdWx0cyBmcm9tIE1vbmdvREIK
+  WxtbMzJtVmVyc2lvbiAwLjYbWzBtXSBTdXBwb3J0IHJldHJpZXZhbCBvZiBp
+  c3N1ZXMsIGlzc3VlIGV2ZW50cyBhbmQgaXNzdWUgaGlzdG9yeQpbG1szMm1W
+  ZXJzaW9uIDAuNhtbMG1dIFN1cHBvcnQgZm9yIHNldHRpbmcgdXNlcm5hbWUv
+  cGFzc3dvcmQgZm9yIHBlcmZvcm1pbmcgcmVxdWVzdHMKWxtbMzJtVmVyc2lv
+  biAwLjYbWzBtXSBSZXNwZWN0IGJ5IGRlZmF1bHQgR2l0aHViJ3MgeC1yYXRl
+  bGltaXQtcmVtYWluaW5nIGhlYWRlcgpbG1szMm1WZXJzaW9uIDAuNhtbMG1d
+  IFNlbGVjdGl2ZSBwcm9jZXNzaW5nIG9mIGV2ZW50cyBmb3IgdXNlci1zcGVj
+  aWZpZWQgcmVwb3MKWxtbMzJtVmVyc2lvbiAwLjYbWzBtXSBOZXcgdG9vbCAo
+  Z2h0LWdldC1tb3JlLWNvbW1pdHMpIHRvIHJldHJpZXZlIGFsbCBjb21taXRz
+  IGZvciBhIHJlcG9zaXRvcnkKWxtbMzJtVmVyc2lvbiAwLjYbWzBtXSBOZXcg
+  dG9vbCAoZ2h0LXByb2Nlc3MtZXZlbnRzKSB0byBwcm9jZXNzIGp1c3Qgb25l
+  IGV2ZW50IGJ5IGlkClsbWzMybVZlcnNpb24gMC42G1swbV0gUmV0cmlldmUg
+  MTAwIGl0ZW1zIGF0IG9uY2UgYnkgZGVmYXVsdCBvbiBtdWx0aXBhZ2UgcmVx
+  dWVzdHMKWxtbMzJtVmVyc2lvbiAwLjYbWzBtXSBSZW5hbWUgd2F0Y2hlcnMg
+  LT4gc3RhcmdhemVycywgYXMgcGVyIEdpdGh1YiBBUEkgY2hhbmdlClsbWzMy
+  bVZlcnNpb24gMC42G1swbV0gRml4ZXMgdG8gYnVncyB0aGF0IHBlcm1pdHRl
+  ZCBlZmZpY2llbnQgcHJvY2Vzc2luZyBvZiBtdWx0aXBhZ2UgcmVxdWVzdHMK
+  WxtbMzJtVmVyc2lvbiAwLjYbWzBtXSBTZXZlcmFsIGZpeGVzIG9uIGhvdyBw
+  dWxsIHJlcXVlc3RzIGFyZSBiZWluZyBwcm9jZXNzZWQKWxtbMzJtVmVyc2lv
+  biAwLjYbWzBtXSBVc2VycyB3aXRoIGludmFsaWQgZ2l0IHNldHVwcyBhcmUg
+  bm93IGFsbG93ZWQKWxtbMzJtVmVyc2lvbiAwLjYbWzBtXSBDb21wYXRpYmls
+  aXR5IHdpdGggUnVieSAxLjggcmVzdG9yZWQK
 rdoc_options:
 - --charset=UTF-8
 require_paths: