ghtorrent 0.10 → 0.11

data/lib/ghtorrent/settings.rb

@@ -24,19 +24,18 @@ module GHTorrent
         :uniq_id => 'mirror.uniq_id',
         :user_agent => 'mirror.user_agent',
 
-        :cache_mode      => 'mirror.cache_mode',
-        :cache_dir       => 'mirror.cache_dir',
-        :cache_stale_age => 'mirror.cache_stale_age',
-
         :github_username => 'mirror.username',
         :github_passwd => 'mirror.passwd',
         :github_token => 'mirror.token',
 
-        :respect_api_ratelimit => 'mirror.respect_api_ratelimit',
-
         :attach_ip => 'mirror.attach_ip',
 
-        :rescue_loops => 'mirror.rescue_loops'
+        :rescue_loops => 'mirror.rescue_loops',
+        :req_limit => 'mirror.req_limit',
+
+        :logging_level => 'logging.level',
+        :logging_uniq => 'logging.uniq',
+        :logging_file => 'logging.file'
     }
 
     DEFAULTS = {
@@ -52,22 +51,20 @@ module GHTorrent
         :mirror_urlbase => 'https://api.github.com/',
         :mirror_persister => 'noop',
         :mirror_history_pages_back => 1,
-        :uniq_id => 'ext_ref_id',
         :user_agent => 'ghtorrent',
 
-        :cache_mode      => 'dev',
-        :cache_dir       => Dir::tmpdir + File::SEPARATOR + 'ghtorrent',
-        :cache_stale_age => 604800,
-
         :github_username => 'foo',
         :github_passwd => 'bar',
         :github_token => '',
 
-        :respect_api_ratelimit => 'true',
-
         :attach_ip => '0.0.0.0',
 
-        :rescue_loops => 'true'
+        :rescue_loops => 'true',
+        :req_limit => 4998,
+
+        :logging_level => 'info',
+        :logging_uniq => '',
+        :logging_file => 'stdout'
     }
 
     def config(key, use_default = true)
@@ -78,7 +75,7 @@ module GHTorrent
         else
           a
         end
-      rescue Exception => e
+      rescue StandardError => e
         if use_default
           DEFAULTS[key]
         else
@@ -102,7 +99,7 @@ module GHTorrent
     end
 
     def settings
-      raise Exception.new('Unimplemented')
+      raise StandardError.new('Unimplemented')
     end
 
   end

data/lib/ghtorrent/{transacted_ghtorrent.rb → transacted_gh_torrent.rb}

@@ -3,12 +3,11 @@ require 'ghtorrent/ghtorrent'
 
 # A version of the GHTorrent class that creates a transaction per processed
 # item
-class TransactedGhtorrent < GHTorrent::Mirror
+class TransactedGHTorrent < GHTorrent::Mirror
 
-  def ensure_repo(owner, repo, commits = false, project_members = false,
-                  watchers = false, forks = false, labels = false)
+  def ensure_repo(owner, repo, recursive = false)
     check_transaction do
-      super(owner, repo, commits, project_members, watchers, forks, labels)
+      super(owner, repo, recursive)
     end
   end
 
@@ -30,6 +29,12 @@ class TransactedGhtorrent < GHTorrent::Mirror
     end
   end
 
+  def ensure_fork_commits(owner, repo, parent_owner, parent_repo)
+    check_transaction do
+      super(owner, repo, parent_owner, parent_repo)
+    end
+  end
+
   def ensure_pull_request(owner, repo, pullreq_id,
                           comments = true, commits = true, history = true,
                           state = nil, actor = nil, created_at = nil)
@@ -86,8 +91,26 @@ class TransactedGhtorrent < GHTorrent::Mirror
     end
   end
 
+  def ensure_user_followers(user)
+    check_transaction do
+      super(user)
+    end
+  end
+
+  def ensure_orgs(user)
+    check_transaction do
+      super(user)
+    end
+  end
+
+  def ensure_org(user, members = true)
+    check_transaction do
+      super(user, members)
+    end
+  end
+
   def check_transaction(&block)
-    if @db.in_transaction?
+    if get_db.in_transaction?
       yield block
     else
       transaction do

data/lib/version.rb

@@ -1,5 +1,5 @@
 module GHTorrent
 
-  VERSION = '0.10'
+  VERSION = '0.11'
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ghtorrent
 version: !ruby/object:Gem::Version
-  version: '0.10'
+  version: '0.11'
 platform: ruby
 authors:
 - Georgios Gousios
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-03 00:00:00.000000000 Z
+date: 2015-09-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mongo
@@ -111,40 +111,6 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: 1.0.0
-- !ruby/object:Gem::Dependency
-  name: rspec
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '2.14'
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 2.14.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '2.14'
-    - - '>='
-      - !ruby/object:Gem::Version
-        version: 2.14.0
-- !ruby/object:Gem::Dependency
-  name: webmock
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '1.16'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ~>
-      - !ruby/object:Gem::Version
-        version: '1.16'
 description: |-
   A library and a collection of associated programs
                       to mirror and process Github data
@@ -153,8 +119,6 @@ executables:
 - ght-data-retrieval
 - ght-mirror-events
 - ght-load
-- ght-rm-dupl
-- ght-process-event
 - ght-get-more-commits
 - ght-retrieve-repo
 - ght-retrieve-user
@@ -167,8 +131,8 @@ files:
 - lib/ghtorrent/adapters/noop_persister.rb
 - lib/ghtorrent/api_client.rb
 - lib/ghtorrent/bson_orderedhash.rb
-- lib/ghtorrent/cache.rb
 - lib/ghtorrent/command.rb
+- lib/ghtorrent/commands/full_user_retriever.rb
 - lib/ghtorrent/commands/ght_data_retrieval.rb
 - lib/ghtorrent/commands/ght_get_more_commits.rb
 - lib/ghtorrent/commands/ght_load.rb
@@ -178,8 +142,9 @@ files:
 - lib/ghtorrent/commands/ght_retrieve_repo.rb
 - lib/ghtorrent/commands/ght_retrieve_repos.rb
 - lib/ghtorrent/commands/ght_retrieve_user.rb
-- lib/ghtorrent/commands/ght_rm_dupl.rb
-- lib/ghtorrent/gh_torrent_exception.rb
+- lib/ghtorrent/commands/ght_retrieve_users.rb
+- lib/ghtorrent/commands/ght_update_repo.rb
+- lib/ghtorrent/event_processing.rb
 - lib/ghtorrent/ghtorrent.rb
 - lib/ghtorrent/hash.rb
 - lib/ghtorrent/logging.rb
@@ -202,35 +167,38 @@ files:
 - lib/ghtorrent/migrations/017_drop_forks_table.rb
 - lib/ghtorrent/migrations/018_drop_merged_user_from_pull_requests.rb
 - lib/ghtorrent/migrations/019_add_fake_to_users.rb
+- lib/ghtorrent/migrations/020_add_deleted_to_users.rb
+- lib/ghtorrent/migrations/021_remove_ext_ref_id.rb
+- lib/ghtorrent/migrations/022_add_project_languages.rb
 - lib/ghtorrent/migrations/mysql_defaults.rb
 - lib/ghtorrent/multiprocess_queue_client.rb
 - lib/ghtorrent/persister.rb
 - lib/ghtorrent/retriever.rb
 - lib/ghtorrent/settings.rb
 - lib/ghtorrent/time.rb
-- lib/ghtorrent/transacted_ghtorrent.rb
+- lib/ghtorrent/transacted_gh_torrent.rb
 - lib/ghtorrent/utils.rb
 - lib/version.rb
 - bin/ght-data-retrieval
 - bin/ght-get-more-commits
 - bin/ght-load
 - bin/ght-log-analyzer
+- bin/ght-log-influx
 - bin/ght-mirror-events
-- bin/ght-process-event
+- bin/ght-queue-grep.rb
 - bin/ght-retrieve-dependents
 - bin/ght-retrieve-one
 - bin/ght-retrieve-repo
 - bin/ght-retrieve-repos
 - bin/ght-retrieve-user
-- bin/ght-rm-dupl
+- bin/ght-retrieve-users
+- bin/ght-update-repo
 - CHANGELOG
 - Gemfile
 - Gemfile.lock
 - LICENSE
 - README.md
 - Rakefile
-- spec/api_client_spec.rb
-- spec/spec_helper.rb
 homepage: https://github.com/gousiosg/github-mirror
 licenses:
 - BSD-2-Clause

data/bin/ght-process-event

@@ -1,35 +0,0 @@
-#!/usr/bin/env ruby
-
-require 'rubygems'
-require 'ghtorrent'
-
-
-class GHTProcessEvent < GHTDataRetrieval
-
-  def prepare_options(options)
-    options.banner <<-BANNER
-Process one or more event ids
-#{command_name} [options] eventid [...]
-    BANNER
-
-  end
-
-  def go
-    ARGV.each do |a|
-      data = ghtorrent.get_event(a)
-      unless data.empty?
-        event = data[0]
-        begin
-          send(event['type'], event)
-        rescue Exception => e
-          puts e
-          puts e.backtrace
-        end
-      else
-        warn "GHTProcessEvent: No event with id #{a}"
-      end
-    end
-  end
-end
-
-GHTProcessEvent.run

data/lib/ghtorrent/cache.rb

@@ -1,97 +0,0 @@
-require 'digest/sha1'
-require 'fileutils'
-
-require 'ghtorrent/logging'
-require 'ghtorrent/settings'
-
-module GHTorrent
-  module Cache
-    include GHTorrent::Logging
-    include GHTorrent::Settings
-
-    # Root dir for cached objects.
-    def cache_dir
-      @cache_dir ||= config(:cache_dir)
-      @cache_dir
-    end
-
-    # The maximum time an item can be cached before being considered stale
-    def max_life
-      @max_life ||= config(:cache_stale_age)
-      @max_life
-    end
-
-    # Put an object to the cache
-    def cache_put(key, object)
-      file = cache_location(key)
-      FileUtils.mkdir_p(File.dirname (file))
-
-      begin
-        File.open(file, 'w') do |f|
-          f.flocked? do
-            YAML::dump object, f
-          end
-        end
-      rescue
-        warn "Cache: Could not cache object #{file} for key #{key}"
-      end
-    end
-
-    # Get the object indexed by +key+ from the cache. Returns nil if the
-    # key is not found or the object is too old.
-    def cache_get(key)
-      file = cache_location(key)
-
-      unless File.exist?(file)
-        return nil
-      end
-
-      unless (Time.now() - File.mtime(file)) < max_life
-        debug "Cache: Cached object for key #{key} too old"
-        return nil
-      end
-
-      begin
-        File.open(file, 'r') do |f|
-          f.flocked? do
-            YAML::load(f)
-          end
-        end
-      rescue
-        warn "Cache: Could not read object from cache location #{file}"
-        File.delete(file)
-      end
-    end
-
-    private
-
-    def cache_location(key)
-      hash = hashkey(key)
-      start = hash[0,2]
-      File.join(cache_dir, start, hash)
-    end
-
-    def hashkey(key)
-      Digest::SHA1.hexdigest key
-    end
-
-  end
-end
-
-class File
-  def flocked? &block
-    status = flock LOCK_EX
-    case status
-      when false
-        return true
-      when 0
-        begin
-          block ? block.call : false
-        ensure
-          flock LOCK_UN
-        end
-      else
-        raise SystemCallError, status
-    end
-  end
-end

data/lib/ghtorrent/commands/ght_rm_dupl.rb

@@ -1,132 +0,0 @@
-require 'rubygems'
-require 'mongo'
-
-require 'ghtorrent/settings'
-require 'ghtorrent/logging'
-require 'ghtorrent/command'
-require 'ghtorrent/persister'
-
-class GHRMDupl < GHTorrent::Command
-
-  include GHTorrent::Settings
-  include GHTorrent::Persister
-
-  def col_info()
-    {
-        :commits => {
-            :unq => "sha",
-            :col => persister.get_underlying_connection.collection(:commits.to_s),
-        },
-        :events => {
-            :unq => "id",
-            :col => persister.get_underlying_connection.collection(:events.to_s),
-        }
-    }
-  end
-
-  def persister
-    @persister ||= connect(:mongo, @settings)
-    @persister
-  end
-
-  def prepare_options(options)
-    options.banner <<-BANNER
-Removes duplicate entries from collections (currently, commits and events)
-
-#{command_name} [options] collection
-
-#{command_name} options:
-    BANNER
-
-    options.opt :earliest, 'Seconds since epoch of earliest item to load',
-                :short => 'e', :default => 0, :type => :int
-    options.opt :snapshot, 'Perform clean up every x records',
-                :short => 's', :default => -1, :type => :int
-  end
-
-  def validate
-    super
-    Trollop::die "no collection specified" unless args[0] && !args[0].empty?
-  end
-
-  # Print MongoDB remove statements that
-  # remove all but one entries for each commit.
-  def remove_duplicates(data, col)
-    removed = 0
-    data.select { |k, v| v.size > 1 }.each do |k, v|
-      v.slice(0..(v.size - 2)).map do |x|
-        removed += 1 if delete_by_id col, x
-      end
-    end
-    removed
-  end
-
-  def delete_by_id(col, id)
-    begin
-      col.remove({'_id' => id})
-      true
-    rescue Mongo::OperationFailure
-      puts "Cannot remove record with id #{id} from #{col.name}"
-      false
-    end
-  end
-
-  def go
-    collection = case ARGV[0]
-                   when "commits" then
-                     :commits
-                   when "events" then
-                     :events
-                   else
-                     puts "Not a known collection name: #{ARGV[0]}\n"
-                 end
-
-    from = {'_id' => {'$gte' => BSON::ObjectId.from_time(Time.at(options[:earliest]))}}
-
-    snapshot = options[:snapshot]
-
-    puts "Deleting duplicates from collection #{collection}"
-    puts "Deleting duplicates after #{Time.at(options[:earliest])}"
-    puts "Perform clean up every #{snapshot} records"
-
-    # Various counters to report stats
-    processed = total_processed = removed = 0
-
-    data = Hash.new
-
-    # The following code needs to save intermediate results to cope
-    # with large datasets
-    col_info[collection][:col].find(from, :fields => col_info[collection][:unq]).each do |r|
-      _id = r["_id"]
-      commit = read_value(r, col_info[collection][:unq])
-
-      # If entries cannot be parsed, remove them
-      if commit.empty?
-        puts "Deleting unknown entry #{_id}"
-        removed += 1 if delete_by_id col_info[collection][:col], _id
-      else
-        data[commit] = [] if data[commit].nil?
-        data[commit] << _id
-      end
-
-      processed += 1
-      total_processed += 1
-
-      print "\rProcessed #{processed} records"
-
-      # Calculate duplicates, save intermediate result
-      if snapshot > 0 and processed > snapshot
-        puts "\nLoaded #{data.size} values, cleaning"
-        removed += remove_duplicates data, col_info[collection][:col]
-        data = Hash.new
-        processed = 0
-      end
-    end
-
-    removed += remove_duplicates data, col_info[collection][:col]
-
-    puts "\nProcessed #{total_processed}, deleted #{removed} duplicates"
-  end
-end
-
-# vim: set sta sts=2 shiftwidth=2 sw=2 et ai :