ghtorrent 0.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/README.md +132 -0
- data/Rakefile +20 -0
- data/bin/ght-data-retrieval +119 -0
- data/bin/ght-load +242 -0
- data/bin/ght-mirror-events +154 -0
- data/bin/ght-periodic-dump +92 -0
- data/bin/ght-rm-dupl +124 -0
- data/bin/ght-torrent-index +180 -0
- data/lib/ghtorrent.rb +22 -0
- data/lib/ghtorrent/adapters/base_adapter.rb +91 -0
- data/lib/ghtorrent/adapters/mongo_persister.rb +126 -0
- data/lib/ghtorrent/adapters/noop_persister.rb +58 -0
- data/lib/ghtorrent/api_client.rb +106 -0
- data/lib/ghtorrent/call_stack.rb +119 -0
- data/lib/ghtorrent/command.rb +136 -0
- data/lib/ghtorrent/ghtorrent.rb +396 -0
- data/lib/ghtorrent/logging.rb +69 -0
- data/lib/ghtorrent/migrations/001_init_schema.rb +60 -0
- data/lib/ghtorrent/migrations/002_add_followers_created_at.rb +15 -0
- data/lib/ghtorrent/migrations/003_add_external_ref_ids.rb +40 -0
- data/lib/ghtorrent/persister.rb +48 -0
- data/lib/ghtorrent/retriever.rb +148 -0
- data/lib/ghtorrent/settings.rb +63 -0
- data/lib/ghtorrent/utils.rb +58 -0
- data/test/callstack_test.rb +67 -0
- metadata +181 -0
@@ -0,0 +1,58 @@
|
|
1
|
+
# Copyright 2012 Georgios Gousios <gousiosg@gmail.com>
|
2
|
+
#
|
3
|
+
# Redistribution and use in source and binary forms, with or
|
4
|
+
# without modification, are permitted provided that the following
|
5
|
+
# conditions are met:
|
6
|
+
#
|
7
|
+
# 1. Redistributions of source code must retain the above
|
8
|
+
# copyright notice, this list of conditions and the following
|
9
|
+
# disclaimer.
|
10
|
+
#
|
11
|
+
# 2. Redistributions in binary form must reproduce the above
|
12
|
+
# copyright notice, this list of conditions and the following
|
13
|
+
# disclaimer in the documentation and/or other materials
|
14
|
+
# provided with the distribution.
|
15
|
+
#
|
16
|
+
# THIS SOFTWARE IS PROVIDED BY BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
|
17
|
+
# AS IS AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
|
18
|
+
# TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
19
|
+
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR
|
20
|
+
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
21
|
+
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
22
|
+
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
|
23
|
+
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
|
24
|
+
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
25
|
+
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
|
26
|
+
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
27
|
+
# POSSIBILITY OF SUCH DAMAGE.
|
28
|
+
|
29
|
+
module GHTorrent
|
30
|
+
module Utils
|
31
|
+
# Read a value whose format is "foo.bar.baz" from a hierarchical map
|
32
|
+
# (the result of a JSON parse or a Mongo query), where a dot represents
|
33
|
+
# one level deep in the result hierarchy.
|
34
|
+
def read_value(from, key)
|
35
|
+
return from if key.nil? or key == ""
|
36
|
+
|
37
|
+
key.split(/\./).reduce({}) do |acc, x|
|
38
|
+
unless acc.nil?
|
39
|
+
if acc.empty?
|
40
|
+
# Initial run
|
41
|
+
acc = from[x]
|
42
|
+
else
|
43
|
+
if acc.has_key?(x)
|
44
|
+
acc = acc[x]
|
45
|
+
else
|
46
|
+
# Some intermediate key does not exist
|
47
|
+
return nil
|
48
|
+
end
|
49
|
+
end
|
50
|
+
else
|
51
|
+
# Some intermediate key returned a null value
|
52
|
+
# This indicates a malformed entry
|
53
|
+
return nil
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
require "test/unit"
|
2
|
+
require 'ghtorrent-old'
|
3
|
+
|
4
|
+
class CallStackTest < Test::Unit::TestCase
|
5
|
+
|
6
|
+
def setup
|
7
|
+
end
|
8
|
+
|
9
|
+
def teardown
|
10
|
+
end
|
11
|
+
|
12
|
+
def test_constructor
|
13
|
+
a = CallStack.new('users', 0)
|
14
|
+
b = CallStack.new('users', 0)
|
15
|
+
assert_equal a,b
|
16
|
+
end
|
17
|
+
|
18
|
+
def test_push
|
19
|
+
stack = CallStack.new('users1', 0)
|
20
|
+
assert_not_nil stack
|
21
|
+
|
22
|
+
stack.push("foo bar")
|
23
|
+
stack.push("2")
|
24
|
+
stack.push("1234421")
|
25
|
+
stack.empty
|
26
|
+
end
|
27
|
+
|
28
|
+
def test_pop
|
29
|
+
stack = CallStack.new('users2', 0)
|
30
|
+
assert_not_nil stack
|
31
|
+
|
32
|
+
stack.push("foo bar")
|
33
|
+
stack.push("2")
|
34
|
+
stack.push("1234421")
|
35
|
+
|
36
|
+
assert stack.pop == "1234421"
|
37
|
+
stack.empty
|
38
|
+
end
|
39
|
+
|
40
|
+
def test_push_pop_push
|
41
|
+
stack = CallStack.new('users3', 0)
|
42
|
+
assert_not_nil stack
|
43
|
+
|
44
|
+
stack.push("foo bar")
|
45
|
+
stack.push("2")
|
46
|
+
|
47
|
+
stack.pop
|
48
|
+
|
49
|
+
stack.push("1234421")
|
50
|
+
|
51
|
+
stack.empty
|
52
|
+
end
|
53
|
+
|
54
|
+
def test_stress
|
55
|
+
stack = CallStack.new('users4', 0)
|
56
|
+
|
57
|
+
1000.times do
|
58
|
+
txt = (0..rand(20)).map{65.+(rand(25)).chr}.join
|
59
|
+
stack.push txt
|
60
|
+
end
|
61
|
+
|
62
|
+
999.times do
|
63
|
+
stack.pop
|
64
|
+
end
|
65
|
+
stack.pop
|
66
|
+
end
|
67
|
+
end
|
metadata
ADDED
@@ -0,0 +1,181 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: ghtorrent
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
hash: 15
|
5
|
+
prerelease:
|
6
|
+
segments:
|
7
|
+
- 0
|
8
|
+
- 2
|
9
|
+
version: "0.2"
|
10
|
+
platform: ruby
|
11
|
+
authors:
|
12
|
+
- Georgios Gousios
|
13
|
+
- Diomidis Spinellis
|
14
|
+
autorequire:
|
15
|
+
bindir: bin
|
16
|
+
cert_chain: []
|
17
|
+
|
18
|
+
date: 2012-05-18 00:00:00 Z
|
19
|
+
dependencies:
|
20
|
+
- !ruby/object:Gem::Dependency
|
21
|
+
name: amqp
|
22
|
+
prerelease: false
|
23
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
24
|
+
none: false
|
25
|
+
requirements:
|
26
|
+
- - ">="
|
27
|
+
- !ruby/object:Gem::Version
|
28
|
+
hash: 25
|
29
|
+
segments:
|
30
|
+
- 0
|
31
|
+
- 9
|
32
|
+
version: "0.9"
|
33
|
+
type: :runtime
|
34
|
+
version_requirements: *id001
|
35
|
+
- !ruby/object:Gem::Dependency
|
36
|
+
name: mongo
|
37
|
+
prerelease: false
|
38
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
39
|
+
none: false
|
40
|
+
requirements:
|
41
|
+
- - ">="
|
42
|
+
- !ruby/object:Gem::Version
|
43
|
+
hash: 3
|
44
|
+
segments:
|
45
|
+
- 1
|
46
|
+
- 6
|
47
|
+
version: "1.6"
|
48
|
+
type: :runtime
|
49
|
+
version_requirements: *id002
|
50
|
+
- !ruby/object:Gem::Dependency
|
51
|
+
name: bson_ext
|
52
|
+
prerelease: false
|
53
|
+
requirement: &id003 !ruby/object:Gem::Requirement
|
54
|
+
none: false
|
55
|
+
requirements:
|
56
|
+
- - ">="
|
57
|
+
- !ruby/object:Gem::Version
|
58
|
+
hash: 3
|
59
|
+
segments:
|
60
|
+
- 1
|
61
|
+
- 6
|
62
|
+
version: "1.6"
|
63
|
+
type: :runtime
|
64
|
+
version_requirements: *id003
|
65
|
+
- !ruby/object:Gem::Dependency
|
66
|
+
name: json
|
67
|
+
prerelease: false
|
68
|
+
requirement: &id004 !ruby/object:Gem::Requirement
|
69
|
+
none: false
|
70
|
+
requirements:
|
71
|
+
- - ">="
|
72
|
+
- !ruby/object:Gem::Version
|
73
|
+
hash: 3
|
74
|
+
segments:
|
75
|
+
- 1
|
76
|
+
- 6
|
77
|
+
version: "1.6"
|
78
|
+
type: :runtime
|
79
|
+
version_requirements: *id004
|
80
|
+
- !ruby/object:Gem::Dependency
|
81
|
+
name: trollop
|
82
|
+
prerelease: false
|
83
|
+
requirement: &id005 !ruby/object:Gem::Requirement
|
84
|
+
none: false
|
85
|
+
requirements:
|
86
|
+
- - ">="
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
hash: 47
|
89
|
+
segments:
|
90
|
+
- 1
|
91
|
+
- 16
|
92
|
+
version: "1.16"
|
93
|
+
type: :runtime
|
94
|
+
version_requirements: *id005
|
95
|
+
- !ruby/object:Gem::Dependency
|
96
|
+
name: sequel
|
97
|
+
prerelease: false
|
98
|
+
requirement: &id006 !ruby/object:Gem::Requirement
|
99
|
+
none: false
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
hash: 65
|
104
|
+
segments:
|
105
|
+
- 3
|
106
|
+
- 35
|
107
|
+
version: "3.35"
|
108
|
+
type: :runtime
|
109
|
+
version_requirements: *id006
|
110
|
+
description: |-
|
111
|
+
A library and a collection of associated programs
|
112
|
+
to mirror and process Github data
|
113
|
+
email: gousiosg@gmail.com
|
114
|
+
executables:
|
115
|
+
- ght-data-retrieval
|
116
|
+
- ght-mirror-events
|
117
|
+
extensions: []
|
118
|
+
|
119
|
+
extra_rdoc_files: []
|
120
|
+
|
121
|
+
files:
|
122
|
+
- lib/ghtorrent/adapters/base_adapter.rb
|
123
|
+
- lib/ghtorrent/adapters/mongo_persister.rb
|
124
|
+
- lib/ghtorrent/adapters/noop_persister.rb
|
125
|
+
- lib/ghtorrent/api_client.rb
|
126
|
+
- lib/ghtorrent/call_stack.rb
|
127
|
+
- lib/ghtorrent/command.rb
|
128
|
+
- lib/ghtorrent/ghtorrent.rb
|
129
|
+
- lib/ghtorrent/logging.rb
|
130
|
+
- lib/ghtorrent/migrations/001_init_schema.rb
|
131
|
+
- lib/ghtorrent/migrations/002_add_followers_created_at.rb
|
132
|
+
- lib/ghtorrent/migrations/003_add_external_ref_ids.rb
|
133
|
+
- lib/ghtorrent/persister.rb
|
134
|
+
- lib/ghtorrent/retriever.rb
|
135
|
+
- lib/ghtorrent/settings.rb
|
136
|
+
- lib/ghtorrent/utils.rb
|
137
|
+
- lib/ghtorrent.rb
|
138
|
+
- bin/ght-data-retrieval
|
139
|
+
- bin/ght-load
|
140
|
+
- bin/ght-mirror-events
|
141
|
+
- bin/ght-periodic-dump
|
142
|
+
- bin/ght-rm-dupl
|
143
|
+
- bin/ght-torrent-index
|
144
|
+
- Rakefile
|
145
|
+
- README.md
|
146
|
+
- test/callstack_test.rb
|
147
|
+
homepage: https://github.com/gousiosg/github-mirror
|
148
|
+
licenses: []
|
149
|
+
|
150
|
+
post_install_message:
|
151
|
+
rdoc_options:
|
152
|
+
- --charset=UTF-8
|
153
|
+
require_paths:
|
154
|
+
- lib
|
155
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
156
|
+
none: false
|
157
|
+
requirements:
|
158
|
+
- - ">="
|
159
|
+
- !ruby/object:Gem::Version
|
160
|
+
hash: 3
|
161
|
+
segments:
|
162
|
+
- 0
|
163
|
+
version: "0"
|
164
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
165
|
+
none: false
|
166
|
+
requirements:
|
167
|
+
- - ">="
|
168
|
+
- !ruby/object:Gem::Version
|
169
|
+
hash: 3
|
170
|
+
segments:
|
171
|
+
- 0
|
172
|
+
version: "0"
|
173
|
+
requirements: []
|
174
|
+
|
175
|
+
rubyforge_project:
|
176
|
+
rubygems_version: 1.8.24
|
177
|
+
signing_key:
|
178
|
+
specification_version: 3
|
179
|
+
summary: Mirror and process Github data
|
180
|
+
test_files: []
|
181
|
+
|