ghtorrent 0.11 → 0.11.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 68b11841a8dfbd0418723fce0620a5d625b4cca1
-  data.tar.gz: d39f30596d257cfe5cb365e1365388169e50cc18
+  metadata.gz: 02d0d3901077bde8352f2cfaf52fcb83aa90fcc4
+  data.tar.gz: ca3d1320d374247884e26444b00a3a460f4101aa
 SHA512:
-  metadata.gz: 6443373ff38703c8113c23716db591a1574eb7f2e36eba1ae68d78ce322b20e42f1c0c840c3a113bb870b7c8b5aae817cb07a60f46d6bccb07ed4058cabc23d2
-  data.tar.gz: cfad88e464fad602f38f6f4cfb7963a0b7ac28f5c205ddf954939937c398d513e004725ef70e920b682b06239c2adc6663c3a29f6eea90913aad3611c9c2a310
+  metadata.gz: 6e1c9d5c24ffa30f9c693eefbc2b0409fbdf2f91ab2314881c5989cad9b716424eeb390b48d1b43f9a900b5fa9556e0f78ccc1ae73e995fcba564a9a1e0c9e57
+  data.tar.gz: 771b2d985ce9869a47bb35231732a72cd3601048ca7b31021396420dfbcdcb106d6408eb539ac9a4750a81984aa2fafe54a7e12e59f28676980e57dd45168c3a

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    ghtorrent (0.11)
+    ghtorrent (0.12)
       bson_ext (~> 1.9, >= 1.9.0)
       bunny (~> 1.0, >= 1.0.0)
       mongo (~> 1.9, >= 1.9.0)

data/bin/ght-update-repo

@@ -3,4 +3,4 @@
 require 'rubygems'
 require 'ghtorrent'
 
-GHTUpdateRepo.run(ARGV)
+GHTRetrieveUser.run(ARGV)

data/lib/ghtorrent.rb

@@ -63,6 +63,5 @@ require 'ghtorrent/commands/ght_retrieve_dependents'
 require 'ghtorrent/commands/ght_retrieve_repos'
 require 'ghtorrent/commands/ght_retrieve_one'
 require 'ghtorrent/commands/ght_retrieve_users'
-require 'ghtorrent/commands/ght_update_repo'
 
 # vim: set sta sts=2 shiftwidth=2 sw=2 et ai :

data/lib/ghtorrent/commands/ght_retrieve_repo.rb

@@ -50,7 +50,7 @@ An efficient way to get all data for a single repo
 
   def stages
     %w(ensure_commits ensure_forks ensure_pull_requests
-       ensure_issues ensure_watchers ensure_labels ensure_languages) #ensure_project_members
+       ensure_issues ensure_watchers ensure_labels) #ensure_project_members
   end
 
 

data/lib/ghtorrent/ghtorrent.rb

@@ -560,7 +560,7 @@ module GHTorrent
       end
 
       functions = %w(ensure_labels ensure_pull_requests
-       ensure_issues ensure_watchers ensure_forks ensure_languages)
+       ensure_issues ensure_watchers ensure_forks)
 
       functions.each do |x|
         send(x, owner, repo)
@@ -568,29 +568,6 @@ module GHTorrent
 
     end
 
-    # Get details about the languages used in the repository
-    def ensure_languages(owner, repo)
-      currepo = ensure_repo(owner, repo)
-      langs = retrieve_languages(owner, repo)
-
-      if langs.nil? or langs.empty?
-        warn "Could not find languages for repo #{owner}/#{repo}"
-        return
-      end
-
-      ts = Time.now
-      langs.keys.each do |lang|
-        @db[:project_languages].insert(
-          :project_id => currepo[:id],
-          :language   => lang.downcase,
-          :lines      => langs[lang],
-          :created_at => ts
-        )
-        info "Added project_language #{owner}/#{repo} -> #{lang} (#{langs[lang]} lines)"
-      end
-      @db[:project_languages].where(:project_id => currepo[:id]).where(:created_at => ts).all
-    end
-
     # Fast path to project forking. Retrieve all commits page by page
     # until we reach a commit that has been registered with the parent
     # repository. Then, copy all remaining parent commits to this repo.

data/lib/ghtorrent/migrations/014_add_deleted_to_projects.rb

@@ -10,7 +10,7 @@ Sequel.migration do
                :default => false
 
     puts 'Field deleted added'
-    puts 'Remember to run the fixes/ght_update_project_info.rb script to mark deleted projects'
+    puts 'Remember to run the fixes/update_deleted.rb script to mark deleted projects'
   end
 
   down do

data/lib/ghtorrent/retriever.rb

@@ -210,10 +210,6 @@ module GHTorrent
       end
     end
 
-    def retrieve_languages(owner, repo)
-      paged_api_request ghurl "repos/#{owner}/#{repo}/languages"
-    end
-
     # Retrieve organizations the provided user participates into
     def retrieve_orgs(user)
       url = ghurl "users/#{user}/orgs"

data/lib/version.rb

@@ -1,5 +1,5 @@
 module GHTorrent
 
-  VERSION = '0.11'
+  VERSION = '0.11.1'
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ghtorrent
 version: !ruby/object:Gem::Version
-  version: '0.11'
+  version: 0.11.1
 platform: ruby
 authors:
 - Georgios Gousios
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-09-24 00:00:00.000000000 Z
+date: 2015-09-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mongo
@@ -143,7 +143,6 @@ files:
 - lib/ghtorrent/commands/ght_retrieve_repos.rb
 - lib/ghtorrent/commands/ght_retrieve_user.rb
 - lib/ghtorrent/commands/ght_retrieve_users.rb
-- lib/ghtorrent/commands/ght_update_repo.rb
 - lib/ghtorrent/event_processing.rb
 - lib/ghtorrent/ghtorrent.rb
 - lib/ghtorrent/hash.rb
@@ -169,7 +168,6 @@ files:
 - lib/ghtorrent/migrations/019_add_fake_to_users.rb
 - lib/ghtorrent/migrations/020_add_deleted_to_users.rb
 - lib/ghtorrent/migrations/021_remove_ext_ref_id.rb
-- lib/ghtorrent/migrations/022_add_project_languages.rb
 - lib/ghtorrent/migrations/mysql_defaults.rb
 - lib/ghtorrent/multiprocess_queue_client.rb
 - lib/ghtorrent/persister.rb

data/lib/ghtorrent/commands/ght_update_repo.rb

@@ -1,126 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'ghtorrent'
-
-class GHTUpdateRepo < GHTorrent::Command
-
-  include GHTorrent::Settings
-  include GHTorrent::Retriever
-  include GHTorrent::Persister
-  include GHTorrent::Logging
-
-  def prepare_options(options)
-    options.banner <<-BANNER
-Updates the deleted field in the project table with current data
-
-#{command_name} owner repo
-
-    BANNER
-  end
-
-  def validate
-    super
-    Trollop::die "Takes two arguments" if ARGV.size == 1
-  end
-
-  def persister
-    @persister ||= connect(:mongo, settings)
-    @persister
-  end
-
-  def db
-    @db ||= @ght.get_db
-  end
-
-  def date(arg)
-    if arg.class != Time
-      Time.parse(arg)#.to_i
-    else
-      arg
-    end
-  end
-
-  def set_deleted(owner, repo)
-    db.from(:projects, :users).\
-       where(:projects__owner_id => :users__id).\
-       where(:users__login => owner).\
-       where(:projects__name => repo).\
-       update(:projects__deleted => true)
-    info("Project #{owner}/#{repo} marked as deleted")
-  end
-
-  def update_mysql(owner, repo, retrieved)
-
-    parent = unless retrieved['parent'].nil?
-               @ght.ensure_repo(retrieved['parent']['owner']['login'],
-                                retrieved['parent']['name'])
-             end
-
-    db.from(:projects, :users).\
-       where(:projects__owner_id => :users__id).\
-       where(:users__login => owner).\
-       where(:projects__name => repo).\
-       update(
-         :projects__url => retrieved['url'],
-         :projects__description => retrieved['description'],
-         :projects__language => retrieved['language'],
-         :projects__created_at => date(retrieved['created_at']),
-         :projects__forked_from => unless parent.nil? then parent[:id] end)
-    debug("Repo #{owner}/#{repo} updated")
-
-    @ght.ensure_languages(owner, repo)
-  end
-
-  def process_project(owner, name)
-    @ght.transaction do
-
-      in_mongo = persister.find(:repos, {'owner.login' => owner, 'name' => name })
-      on_github = api_request(ghurl ("repos/#{owner}/#{name}"))
-
-      unless in_mongo.empty? and on_github.empty?
-        in_mysql = retrieve_repo(owner, name)
-      end
-
-      if in_mongo.empty?
-        if on_github.empty?
-          if in_mysql.nil?
-            # Project does not exist anywhere
-            warn "Repo #{owner}/#{name} does not exist in MySQL"
-          else
-            # Project exists in MySQL but not on Github or Mongo
-            # Mark it as deleted
-            set_deleted(owner, name)
-          end
-        else
-          # Project does not exist in Mongo, but exists in Github
-          if in_mysql.nil?
-            warn "Repo #{owner}/#{name} does not exist in MySQL"
-          else
-            # The retrieval process already added it to Mongo, so update MySQL
-            update_mysql(owner, name, in_mysql)
-          end
-        end
-      else
-        if on_github.empty?
-          # Project was deleted on Github. Mark it as deleted.
-          set_deleted(owner, name)
-        else
-          update_mysql(owner, name, in_mysql)
-        end
-      end
-    end
-  end
-
-  def go
-
-    @ght ||= GHTorrent::Mirror.new(settings)
-
-    unless ARGV[1].nil?
-      process_project(ARGV[0], ARGV[1])
-      exit(0)
-    end
-
-  end
-end
-
-GHTUpdateRepo.run

data/lib/ghtorrent/migrations/022_add_project_languages.rb

@@ -1,24 +0,0 @@
-require 'sequel'
-
-require 'ghtorrent/migrations/mysql_defaults'
-
-
-Sequel.migration do
-
-  up do
-    puts 'Adding table project_languages'
-
-    create_table :project_languages do
-      foreign_key :project_id, :projects, :null => false
-      String :language, :null => false
-      Integer :lines, :null => false, :default => 0
-      DateTime :created_at, :null => false,
-               :default => Sequel::CURRENT_TIMESTAMP
-    end
-  end
-
-  down do
-    puts 'Dropping table project_languages'
-    drop_table :project_languages
-  end
-end