ghtorrent 0.4 → 0.5

data/lib/ghtorrent/settings.rb

@@ -1,5 +1,7 @@
 require 'yaml'
 
+require 'ghtorrent/utils'
+
 module GHTorrent
   module Settings
 
@@ -11,6 +13,7 @@ module GHTorrent
         :amqp_username => "amqp.username",
         :amqp_password => "amqp.password",
         :amqp_exchange => "amqp.exchange",
+        :amqp_prefetch  => "amqp.prefetch",
 
         :sql_url => "sql.url",
 
@@ -21,7 +24,13 @@ module GHTorrent
         :mirror_persister => "mirror.persister",
         :mirror_commit_pages_new_repo => "mirror.commit_pages_new_repo",
 
-        :uniq_id => "mirror.uniq_id"
+        :uniq_id => "mirror.uniq_id",
+
+        :cache_mode      => "mirror.cache_mode",
+        :cache_dir       => "mirror.cache_dir",
+        :cache_stale_age => "mirror.cache_stale_age",
+
+        :attach_ip => "mirror.attach_ip"
     }
 
     def config(key)
@@ -32,5 +41,15 @@ module GHTorrent
       more_keys.each {|k,v| CONFIGKEYS[k] = v}
     end
 
+    def merge_config_values(values)
+      values.reduce(settings) {|acc, k|
+        acc.merge_recursive write_value(settings, CONFIGKEYS[k[0]], k[1])
+      }
+    end
+
+    def settings
+      raise Exception("Unimplemented")
+    end
+
   end
 end

data/lib/ghtorrent/time.rb

@@ -0,0 +1,5 @@
+class Time
+  def to_ms
+    (self.to_f * 1000.0).to_i
+  end
+end

data/lib/ghtorrent/utils.rb

@@ -1,3 +1,5 @@
+require 'ghtorrent/hash'
+
 module GHTorrent
   module Utils
 
@@ -5,9 +7,8 @@ module GHTorrent
       other.extend self
     end
 
-    # Read a value whose format is "foo.bar.baz" from a hierarchical map
-    # (the result of a JSON parse or a Mongo query), where a dot represents
-    # one level deep in the result hierarchy.
+    # Read the value for a key whose format is "foo.bar.baz" from a hierarchical
+    # map, where a dot represents one level deep in the hierarchy.
     def read_value(from, key)
       return from if key.nil? or key == ""
 
@@ -32,6 +33,23 @@ module GHTorrent
       end
     end
 
+    # Overwrite an existing +key+ whose format is "foo.bar" (where a dot
+    # represents one level deep in the hierarchy) in hash +to+ with +value+.
+    # If the key does not exist, it will be added at the appropriate depth level
+    def write_value(to, key, value)
+      return to if key.nil? or key == ""
+
+      prev = nil
+      key.split(/\./).reverse.each {|x|
+        a = Hash.new
+        a[x] = if prev.nil? then value else prev end
+        prev = a
+        a
+      }
+
+      to.merge_recursive(prev)
+    end
+
     def user_type(type)
       if type == "User"
         "USR"
@@ -40,4 +58,4 @@ module GHTorrent
       end
     end
   end
-end
+end

data/lib/version.rb

@@ -0,0 +1,5 @@
+module GHTorrent
+
+  VERSION = '0.5'
+
+end

metadata

@@ -1,163 +1,166 @@
---- !ruby/object:Gem::Specification 
+--- !ruby/object:Gem::Specification
 name: ghtorrent
-version: !ruby/object:Gem::Version 
-  hash: 3
+version: !ruby/object:Gem::Version
+  version: '0.5'
   prerelease: 
-  segments: 
-  - 0
-  - 4
-  version: "0.4"
 platform: ruby
-authors: 
+authors:
 - Georgios Gousios
 - Diomidis Spinellis
 autorequire: 
 bindir: bin
 cert_chain: []
-
-date: 2012-07-11 00:00:00 Z
-dependencies: 
-- !ruby/object:Gem::Dependency 
+date: 2012-08-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
   name: amqp
-  prerelease: false
-  requirement: &id001 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 25
-        segments: 
-        - 0
-        - 9
-        version: "0.9"
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0.9'
   type: :runtime
-  version_requirements: *id001
-- !ruby/object:Gem::Dependency 
-  name: mongo
   prerelease: false
-  requirement: &id002 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 1
-        - 6
-        version: "1.6"
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0.9'
+- !ruby/object:Gem::Dependency
+  name: mongo
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '1.6'
   type: :runtime
-  version_requirements: *id002
-- !ruby/object:Gem::Dependency 
-  name: bson_ext
   prerelease: false
-  requirement: &id003 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 1
-        - 6
-        version: "1.6"
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: bson_ext
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '1.6'
   type: :runtime
-  version_requirements: *id003
-- !ruby/object:Gem::Dependency 
-  name: json
   prerelease: false
-  requirement: &id004 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 1
-        - 6
-        version: "1.6"
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '1.6'
   type: :runtime
-  version_requirements: *id004
-- !ruby/object:Gem::Dependency 
-  name: trollop
   prerelease: false
-  requirement: &id005 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 47
-        segments: 
-        - 1
-        - 16
-        version: "1.16"
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '1.6'
+- !ruby/object:Gem::Dependency
+  name: trollop
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '1.16'
   type: :runtime
-  version_requirements: *id005
-- !ruby/object:Gem::Dependency 
-  name: sequel
   prerelease: false
-  requirement: &id006 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 65
-        segments: 
-        - 3
-        - 35
-        version: "3.35"
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: sequel
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '3.35'
   type: :runtime
-  version_requirements: *id006
-- !ruby/object:Gem::Dependency 
-  name: sqlite3-ruby
   prerelease: false
-  requirement: &id007 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 31
-        segments: 
-        - 1
-        - 3
-        - 2
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '3.35'
+- !ruby/object:Gem::Dependency
+  name: sqlite3-ruby
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
         version: 1.3.2
   type: :runtime
-  version_requirements: *id007
-- !ruby/object:Gem::Dependency 
-  name: daemons
   prerelease: false
-  requirement: &id008 !ruby/object:Gem::Requirement 
-    none: false
-    requirements: 
-    - - ">="
-      - !ruby/object:Gem::Version 
-        hash: 3
-        segments: 
-        - 1
-        - 1
-        - 8
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.3.2
+- !ruby/object:Gem::Dependency
+  name: daemons
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
         version: 1.1.8
   type: :runtime
-  version_requirements: *id008
-description: |-
-  A library and a collection of associated programs
-                      to mirror and process Github data
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 1.1.8
+description: ! "A library and a collection of associated programs\n                    to
+  mirror and process Github data"
 email: gousiosg@gmail.com
-executables: 
+executables:
 - ght-data-retrieval
 - ght-mirror-events
+- ght-load
+- ght-rm-dupl
 extensions: []
-
 extra_rdoc_files: []
-
-files: 
+files:
 - lib/ghtorrent/adapters/base_adapter.rb
 - lib/ghtorrent/adapters/mongo_persister.rb
 - lib/ghtorrent/adapters/noop_persister.rb
 - lib/ghtorrent/api_client.rb
-- lib/ghtorrent/call_stack.rb
+- lib/ghtorrent/bson_orderedhash.rb
+- lib/ghtorrent/cache.rb
 - lib/ghtorrent/command.rb
+- lib/ghtorrent/gh_torrent_exception.rb
 - lib/ghtorrent/ghtorrent.rb
+- lib/ghtorrent/hash.rb
 - lib/ghtorrent/logging.rb
 - lib/ghtorrent/migrations/001_init_schema.rb
 - lib/ghtorrent/migrations/002_add_external_ref_ids.rb
@@ -165,53 +168,78 @@ files:
 - lib/ghtorrent/migrations/004_add_commit_comments.rb
 - lib/ghtorrent/migrations/005_add_repo_collaborators.rb
 - lib/ghtorrent/migrations/006_add_watchers.rb
+- lib/ghtorrent/migrations/007_add_pull_requests.rb
+- lib/ghtorrent/migrations/008_add_project_unq.rb
+- lib/ghtorrent/migrations/009_add_project_commit.rb
+- lib/ghtorrent/migrations/010_add_forks.rb
+- lib/ghtorrent/migrations/mysql_defaults.rb
 - lib/ghtorrent/persister.rb
 - lib/ghtorrent/retriever.rb
 - lib/ghtorrent/settings.rb
+- lib/ghtorrent/time.rb
 - lib/ghtorrent/utils.rb
 - lib/ghtorrent.rb
+- lib/version.rb
 - bin/ght-data-retrieval
 - bin/ght-load
 - bin/ght-mirror-events
 - bin/ght-periodic-dump
 - bin/ght-rm-dupl
 - bin/ght-torrent-index
+- CHANGELOG
+- Gemfile
+- Gemfile.lock
 - LICENSE
 - Rakefile
 - README.md
 - test/callstack_test.rb
 homepage: https://github.com/gousiosg/github-mirror
 licenses: []
-
-post_install_message: 
-rdoc_options: 
+post_install_message: !binary |-
+  WxtbMzJtVmVyc2lvbiAwLjUbWzBtXSBHZW5lcmljIG1ldGhvZHMgZm9yIHJl
+  dHJpZXZpbmcgaXRlbXMgdGhhdCBhcmUgYm91bmQgdG8gcmVwb3NpdG9yaWVz
+  ClsbWzMybVZlcnNpb24gMC41G1swbV0gUHJvY2Vzc2luZyBvZiBwdWxsIHJl
+  cXVlc3RzIHdpdGggY29tbWl0cywgY29tbWVudHMgYW5kIGhpc3RvcnkKWxtb
+  MzJtVmVyc2lvbiAwLjUbWzBtXSBQcm9jZXNzaW5nIG9mIHByb2plY3QgZm9y
+  a3MKWxtbMzJtVmVyc2lvbiAwLjUbWzBtXSBOZXcgdG9vbCAoZ2h0LWxvYWQp
+  IHRvIGZpbHRlciBhbmQgbG9hZCBldmVudHMgdG8gdGhlIHF1ZXVlClsbWzMy
+  bVZlcnNpb24gMC41G1swbV0gTmV3IHRvb2wgKGdodC1ybS1kdXBsKSB0byBk
+  ZWxldGUgZHVwbGljYXRlIGVudHJpZXMgZnJvbSBjb2xsZWN0aW9ucyAoZXZl
+  bnRzICYgY29tbWl0cyBzdXBwb3J0ZWQpClsbWzMybVZlcnNpb24gMC41G1sw
+  bV0gUHJvamVjdCB3aWRlIHJlcXVlc3RpbmcgcmVzdWx0IGNhY2hpbmcgZm9y
+  IG11bHRpLXBhZ2UgcmVxdWVzdHMKWxtbMzJtVmVyc2lvbiAwLjUbWzBtXSBC
+  ZXR0ZXIgbG9nZ2luZyBpbiB2YXJpb3VzIHBsYWNlcwpbG1szMm1WZXJzaW9u
+  IDAuNRtbMG1dIEJldHRlciBkZWZhdWx0cyBmb3IgTXlTUUwgKFVURjggKyBJ
+  bm5vREIgdGFibGVzKQpbG1szMm1WZXJzaW9uIDAuNRtbMG1dIENvbW1pdHMg
+  YXJlIG5vdyBzZXBlcmF0ZWQgZnJvbSBwcm9qZWN0cy4gUHJvamVjdCBmb3Jr
+  cyBjYW4gc2hhcmUgY29tbWl0cy4KWxtbMzJtVmVyc2lvbiAwLjUbWzBtXSBT
+  dXBwb3J0IGZvciBzZXR0aW5nIHRoZSBJUCBhZGRyZXNzIHRvIHVzZSBmb3Ig
+  cmV0cmlldmFsIG9uIG11bHRpLWhvbWVkClsbWzMybVZlcnNpb24gMC41G1sw
+  bV0gQ29tcGF0aWJpbGl0eSB3aXRoIFJ1YnkgMS45IChub3cgZGVmYXVsdCkg
+  YW5kIEpSdWJ5ClsbWzMybVZlcnNpb24gMC41G1swbV0gUHJvcGVyIG1vZHVs
+  YXJpemF0aW9uLCBmb2xsb3dpbmcgdGhlIGNha2UgZGVzaWduIHBhdHRlcm4K
+  WxtbMzJtVmVyc2lvbiAwLjUbWzBtXSBOZXZlciByZXRyaWV2ZSBhcnJheXMg
+  b2YgcmVzdWx0cyBmcm9tIE1vbmdvREIK
+rdoc_options:
 - --charset=UTF-8
-require_paths: 
+require_paths:
 - lib
-required_ruby_version: !ruby/object:Gem::Requirement 
+required_ruby_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
-required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
   none: false
-  requirements: 
-  - - ">="
-    - !ruby/object:Gem::Version 
-      hash: 3
-      segments: 
-      - 0
-      version: "0"
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
 requirements: []
-
 rubyforge_project: 
 rubygems_version: 1.8.24
 signing_key: 
 specification_version: 3
 summary: Mirror and process Github data
 test_files: []
-