ghtorrent 0.8.1 → 0.9
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +13 -5
- data/CHANGELOG +3 -0
- data/Gemfile.lock +4 -6
- data/README.md +1 -1
- data/lib/ghtorrent/command.rb +62 -1
- data/lib/ghtorrent/commands/ght_data_retrieval.rb +64 -57
- data/lib/ghtorrent/commands/ght_load.rb +24 -50
- data/lib/ghtorrent/commands/ght_mirror_events.rb +27 -39
- data/lib/ghtorrent/commands/ght_retrieve_dependents.rb +0 -1
- data/lib/ghtorrent/commands/ght_retrieve_repos.rb +28 -61
- data/lib/version.rb +1 -1
- metadata +22 -23
checksums.yaml
CHANGED
@@ -1,7 +1,15 @@
|
|
1
1
|
---
|
2
|
-
|
3
|
-
metadata.gz:
|
4
|
-
|
2
|
+
!binary "U0hBMQ==":
|
3
|
+
metadata.gz: !binary |-
|
4
|
+
MzBhZTlkNGYxYTc3ZTg4NzQ3OTQyMWQ3NTMxZGY1YzljYmE0MzIwZg==
|
5
|
+
data.tar.gz: !binary |-
|
6
|
+
NmE3OGNlNGY4Y2QzMGJkY2E3NWY0MTNhMTU2MDAyYTA3MDUzN2IzZQ==
|
5
7
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
|
8
|
+
metadata.gz: !binary |-
|
9
|
+
MTkwZWQwNmQ2MjE1OTU5ZWQ5NWRmZWFhYjY4ZGM3NTA5ZmY2ZDcyMTI0OGZj
|
10
|
+
ZGY0MDczZGFmYTZlZTEyYmMxYWI4YTU5ODQyZGFmM2YwZmNhODljMDZmMWJk
|
11
|
+
ZThiYWQ5OGUxMmI3Yjg1Nzk1MjBkZTFiNzkxZmE1NTAzNGFmNzM=
|
12
|
+
data.tar.gz: !binary |-
|
13
|
+
MmJiODg3NmU2OTdjZTA2Njc0MmFmZWRhM2JkZjA4NjY5OTRiZTJhOTEzZGJk
|
14
|
+
NmJhN2JkNGFjODdhZDk0YWY4MDIzY2NkODU5MDhjZDQxNjA4NzU0ZWFkYmMx
|
15
|
+
ZDE2Y2U3NzQ0ZDAxN2YxNmNjZDRiZTUzMDQ5OWY5OWY3ZDc3Yjc=
|
data/CHANGELOG
CHANGED
data/Gemfile.lock
CHANGED
@@ -2,8 +2,8 @@ PATH
|
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
4
|
ghtorrent (0.8.1)
|
5
|
-
amqp (~> 1.1.0)
|
6
5
|
bson_ext (~> 1.9.0)
|
6
|
+
bunny (~> 1.0.0)
|
7
7
|
mongo (~> 1.9.0)
|
8
8
|
sequel (~> 4.5.0)
|
9
9
|
trollop (~> 2.0.0)
|
@@ -12,17 +12,15 @@ GEM
|
|
12
12
|
remote: https://rubygems.org/
|
13
13
|
specs:
|
14
14
|
addressable (2.3.5)
|
15
|
-
amq-protocol (1.9.
|
16
|
-
amqp (1.1.7)
|
17
|
-
amq-protocol (>= 1.9.0)
|
18
|
-
eventmachine
|
15
|
+
amq-protocol (1.9.2)
|
19
16
|
bson (1.9.2)
|
20
17
|
bson_ext (1.9.2)
|
21
18
|
bson (~> 1.9.2)
|
19
|
+
bunny (1.0.7)
|
20
|
+
amq-protocol (>= 1.9.2)
|
22
21
|
crack (0.4.1)
|
23
22
|
safe_yaml (~> 0.9.0)
|
24
23
|
diff-lcs (1.2.5)
|
25
|
-
eventmachine (1.0.3)
|
26
24
|
mongo (1.9.2)
|
27
25
|
bson (~> 1.9.2)
|
28
26
|
rspec (2.14.1)
|
data/README.md
CHANGED
@@ -8,7 +8,7 @@ checking out this repository.
|
|
8
8
|
GHTorrent can be used for a variety of purposes, such as:
|
9
9
|
|
10
10
|
* Mirror the Github API event stream and follow links from events to actual data
|
11
|
-
to gradually build a [Github index](http://ghtorrent.org/
|
11
|
+
to gradually build a [Github index](http://ghtorrent.org/)
|
12
12
|
* Create a queriable metadata index for a specific repository
|
13
13
|
* Query the Github API using intelligent caching to avoid duplicate queries
|
14
14
|
|
data/lib/ghtorrent/command.rb
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
require 'rubygems'
|
2
2
|
require 'trollop'
|
3
|
-
require '
|
3
|
+
require 'bunny'
|
4
4
|
require 'etc'
|
5
5
|
|
6
6
|
require 'ghtorrent/settings'
|
@@ -135,6 +135,67 @@ Standard options:
|
|
135
135
|
def go
|
136
136
|
end
|
137
137
|
|
138
|
+
# Specify a handler to incoming messages from a connection to
|
139
|
+
# a queue.
|
140
|
+
# [queue]: The queue name to bind to
|
141
|
+
# [ack]: :before or :after when should acks be send, before or after
|
142
|
+
# the block returns
|
143
|
+
# [block]: A block with one argument (the message)
|
144
|
+
def queue_client(queue, ack = :after, block)
|
145
|
+
|
146
|
+
stopped = false
|
147
|
+
while not stopped
|
148
|
+
begin
|
149
|
+
conn = Bunny.new(:host => config(:amqp_host),
|
150
|
+
:port => config(:amqp_port),
|
151
|
+
:username => config(:amqp_username),
|
152
|
+
:password => config(:amqp_password))
|
153
|
+
conn.start
|
154
|
+
|
155
|
+
ch = conn.create_channel
|
156
|
+
debug "Setting prefetch to #{config(:amqp_prefetch)}"
|
157
|
+
ch.prefetch(config(:amqp_prefetch))
|
158
|
+
debug "Connection to #{config(:amqp_host)} succeded"
|
159
|
+
|
160
|
+
x = ch.topic(config(:amqp_exchange), :durable => true,
|
161
|
+
:auto_delete => false)
|
162
|
+
q = ch.queue(queue, :durable => true)
|
163
|
+
q.bind(x)
|
164
|
+
|
165
|
+
q.subscribe(:block => true,
|
166
|
+
:ack => true) do |delivery_info, properties, msg|
|
167
|
+
|
168
|
+
if ack == :before
|
169
|
+
ch.acknowledge(delivery_info.delivery_tag, false)
|
170
|
+
end
|
171
|
+
|
172
|
+
begin
|
173
|
+
block.call(msg)
|
174
|
+
ensure
|
175
|
+
ch.acknowledge(delivery_info.delivery_tag, false)
|
176
|
+
end
|
177
|
+
end
|
178
|
+
|
179
|
+
rescue Bunny::TCPConnectionFailed => e
|
180
|
+
warn "Connection to #{config(:amqp_host)} failed. Retrying in 1 sec"
|
181
|
+
sleep(1)
|
182
|
+
rescue Bunny::PossibleAuthenticationFailureError => e
|
183
|
+
warn "Could not authenticate as #{conn.username}"
|
184
|
+
rescue Bunny::NotFound, Bunny::AccessRefused, Bunny::PreconditionFailed => e
|
185
|
+
warn "Channel error: #{e}. Retrying in 1 sec"
|
186
|
+
sleep(1)
|
187
|
+
rescue Interrupt => _
|
188
|
+
stopped = true
|
189
|
+
rescue Exception => e
|
190
|
+
raise e
|
191
|
+
end
|
192
|
+
end
|
193
|
+
|
194
|
+
ch.close unless ch.nil?
|
195
|
+
conn.close unless conn.nil?
|
196
|
+
|
197
|
+
end
|
198
|
+
|
138
199
|
def override_config(config_file, setting, new_value)
|
139
200
|
puts "Overriding configuration #{setting}=#{config(setting)} with cmd line #{new_value}"
|
140
201
|
merge_config_values(config_file, {setting => new_value})
|
@@ -1,7 +1,6 @@
|
|
1
1
|
require 'rubygems'
|
2
|
-
require '
|
2
|
+
require 'bunny'
|
3
3
|
require 'json'
|
4
|
-
require 'pp'
|
5
4
|
|
6
5
|
require 'ghtorrent/ghtorrent'
|
7
6
|
require 'ghtorrent/settings'
|
@@ -158,67 +157,75 @@ Retrieves events from queues and processes them through GHTorrent
|
|
158
157
|
}
|
159
158
|
end
|
160
159
|
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
172
|
-
|
173
|
-
|
174
|
-
|
175
|
-
|
176
|
-
|
177
|
-
|
178
|
-
|
179
|
-
|
180
|
-
|
181
|
-
|
182
|
-
|
183
|
-
|
184
|
-
|
185
|
-
|
186
|
-
|
187
|
-
|
188
|
-
|
189
|
-
|
190
|
-
|
191
|
-
|
192
|
-
event.delete '_id'
|
193
|
-
data = parse(event.to_json)
|
194
|
-
info "GHTDataRetrieval: Processing event: #{data['type']}-#{data['id']}"
|
195
|
-
|
196
|
-
unless options[:filter].nil?
|
197
|
-
if filter.include?(data['repo']['name'])
|
198
|
-
send(h, data)
|
199
|
-
else
|
200
|
-
info "GHTDataRetrieval: Repo #{data['repo']['name']} not in process list. Ignoring event #{data['type']}-#{data['id']}"
|
201
|
-
end
|
202
|
-
else
|
160
|
+
|
161
|
+
conn = Bunny.new(:host => config(:amqp_host),
|
162
|
+
:port => config(:amqp_port),
|
163
|
+
:username => config(:amqp_username),
|
164
|
+
:password => config(:amqp_password))
|
165
|
+
conn.start
|
166
|
+
|
167
|
+
channel = conn.create_channel
|
168
|
+
debug "Setting prefetch to #{config(:amqp_prefetch)}"
|
169
|
+
channel.prefetch(config(:amqp_prefetch))
|
170
|
+
debug "Connection to #{config(:amqp_host)} succeded"
|
171
|
+
|
172
|
+
exchange = channel.topic(config(:amqp_exchange), :durable => true,
|
173
|
+
:auto_delete => false)
|
174
|
+
|
175
|
+
handlers.each do |h|
|
176
|
+
queue = channel.queue("#{h}s", {:durable => true})\
|
177
|
+
.bind(exchange, :routing_key => "evt.#{h}")
|
178
|
+
|
179
|
+
info "GHTDataRetrieval: Binding handler #{h} to routing key evt.#{h}"
|
180
|
+
|
181
|
+
queue.subscribe(:ack => true) do |headers, properties, msg|
|
182
|
+
begin
|
183
|
+
|
184
|
+
event = persister.get_underlying_connection[:events].find_one('id' => msg)
|
185
|
+
event.delete '_id'
|
186
|
+
data = parse(event.to_json)
|
187
|
+
info "GHTDataRetrieval: Processing event: #{data['type']}-#{data['id']}"
|
188
|
+
|
189
|
+
unless options[:filter].nil?
|
190
|
+
if filter.include?(data['repo']['name'])
|
203
191
|
send(h, data)
|
204
|
-
end
|
205
|
-
headers.ack
|
206
|
-
info "GHTDataRetrieval: Processed event: #{data['type']}-#{data['id']}"
|
207
|
-
rescue Exception => e
|
208
|
-
# Give a message a chance to be reprocessed
|
209
|
-
if headers.redelivered?
|
210
|
-
warn "GHTDataRetrieval: Could not process event: #{msg}"
|
211
|
-
headers.reject(:requeue => false)
|
212
192
|
else
|
213
|
-
|
193
|
+
info "GHTDataRetrieval: Repo #{data['repo']['name']} not in process list. Ignoring event #{data['type']}-#{data['id']}"
|
214
194
|
end
|
215
|
-
|
216
|
-
|
217
|
-
|
195
|
+
else
|
196
|
+
send(h, data)
|
197
|
+
end
|
198
|
+
channel.acknowledge(headers.delivery_tag, false)
|
199
|
+
info "GHTDataRetrieval: Processed event: #{data['type']}-#{data['id']}"
|
200
|
+
rescue Exception => e
|
201
|
+
# Give a message a chance to be reprocessed
|
202
|
+
if headers.redelivered?
|
203
|
+
warn "GHTDataRetrieval: Could not process event: #{msg}"
|
204
|
+
channel.reject(headers.delivery_tag, false)
|
205
|
+
else
|
206
|
+
channel.reject(headers.delivery_tag, true)
|
218
207
|
end
|
208
|
+
|
209
|
+
STDERR.puts e
|
210
|
+
STDERR.puts e.backtrace.join("\n")
|
219
211
|
end
|
220
|
-
|
212
|
+
end
|
221
213
|
end
|
214
|
+
|
215
|
+
stopped = false
|
216
|
+
while not stopped
|
217
|
+
begin
|
218
|
+
sleep(1)
|
219
|
+
rescue Interrupt => _
|
220
|
+
debug 'Exit requested'
|
221
|
+
stopped = true
|
222
|
+
end
|
223
|
+
end
|
224
|
+
|
225
|
+
debug 'Closing AMQP connection'
|
226
|
+
channel.close unless channel.nil?
|
227
|
+
conn.close unless conn.nil?
|
228
|
+
|
222
229
|
end
|
223
230
|
end
|
224
231
|
|
@@ -1,8 +1,7 @@
|
|
1
1
|
require 'rubygems'
|
2
2
|
require 'mongo'
|
3
|
-
require 'amqp'
|
4
|
-
require 'eventmachine'
|
5
3
|
require 'pp'
|
4
|
+
require 'bunny'
|
6
5
|
|
7
6
|
require 'ghtorrent/settings'
|
8
7
|
require 'ghtorrent/logging'
|
@@ -36,8 +35,6 @@ Loads object ids from a collection to a queue for further processing.
|
|
36
35
|
:type => :int
|
37
36
|
options.opt :number, 'Total number of items to load',
|
38
37
|
:short => 'n', :type => :int, :default => 2**48
|
39
|
-
options.opt :batch, 'Number of items to process in a batch',
|
40
|
-
:short => 'b', :type => :int, :default => 10000
|
41
38
|
options.opt :filter,
|
42
39
|
'Filter items by regexp on item attributes: item.attr=regexp',
|
43
40
|
:short => 'f', :type => String, :multi => true
|
@@ -62,16 +59,10 @@ Loads object ids from a collection to a queue for further processing.
|
|
62
59
|
# Num events read
|
63
60
|
total_read = 0
|
64
61
|
|
65
|
-
puts "Loading
|
66
|
-
puts "Loading
|
67
|
-
puts "Loading #{options[:batch]} items per batch" if options[:batch]
|
62
|
+
puts "Loading events after #{Time.at(options[:earliest])}" if options[:verbose]
|
63
|
+
puts "Loading events before #{Time.at(options[:latest])}" if options[:verbose]
|
68
64
|
puts "Loading #{options[:number]} items" if options[:verbose]
|
69
65
|
|
70
|
-
if options[:batch] >= options[:number]
|
71
|
-
puts "Batch > number of items, setting batch to #{options[:number]}"
|
72
|
-
options[:batch] = options[:number]
|
73
|
-
end
|
74
|
-
|
75
66
|
what = case
|
76
67
|
when options[:filter].is_a?(Array)
|
77
68
|
options[:filter].reduce({}) { |acc,x|
|
@@ -90,60 +81,43 @@ Loads object ids from a collection to a queue for further processing.
|
|
90
81
|
|
91
82
|
(puts 'Mongo filter:'; pp what.merge(from)) if options[:verbose]
|
92
83
|
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
84
|
+
conn = Bunny.new(:host => config(:amqp_host),
|
85
|
+
:port => config(:amqp_port),
|
86
|
+
:username => config(:amqp_username),
|
87
|
+
:password => config(:amqp_password))
|
88
|
+
conn.start
|
97
89
|
|
98
|
-
|
99
|
-
|
100
|
-
:durable => true, :auto_delete => false)
|
90
|
+
channel = conn.create_channel
|
91
|
+
puts "Connection to #{config(:amqp_host)} succeded"
|
101
92
|
|
102
|
-
|
103
|
-
|
104
|
-
puts('Closing connection')
|
105
|
-
connection.close { EventMachine.stop }
|
106
|
-
}
|
93
|
+
exchange = channel.topic(config(:amqp_exchange),
|
94
|
+
:durable => true, :auto_delete => false)
|
107
95
|
|
108
|
-
|
109
|
-
|
110
|
-
|
96
|
+
stopped = false
|
97
|
+
while not stopped
|
98
|
+
begin
|
111
99
|
persister.get_underlying_connection[:events].find(what.merge(from),
|
112
|
-
|
113
|
-
:skip => total_read,
|
114
|
-
:limit => options[:batch]).each do |e|
|
100
|
+
:snapshot => true).each do |e|
|
115
101
|
unq = read_value(e, 'type')
|
116
102
|
if unq.class != String or unq.nil? then
|
117
|
-
|
103
|
+
raise Exception.new('Unique value can only be a String')
|
118
104
|
end
|
119
105
|
|
120
106
|
exchange.publish e['id'], :persistent => false,
|
121
107
|
:routing_key => "evt.#{e['type']}"
|
122
108
|
|
123
|
-
num_read += 1
|
124
109
|
total_read += 1
|
125
|
-
puts "Publish id = #{e['id']} (#{
|
126
|
-
end
|
110
|
+
puts "Publish id = #{e['id']} (#{total_read} read)" if options.verbose
|
127
111
|
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
else
|
132
|
-
# Schedule new event processing cycle
|
133
|
-
EventMachine.next_tick do
|
134
|
-
read_and_publish.call
|
112
|
+
if total_read >= options[:number]
|
113
|
+
puts 'Finished reading, exiting'
|
114
|
+
break
|
135
115
|
end
|
136
116
|
end
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
Signal.trap('INT', show_stopper)
|
141
|
-
Signal.trap('TERM', show_stopper)
|
142
|
-
|
143
|
-
EventMachine.add_timer(0.1) do
|
144
|
-
read_and_publish.call
|
117
|
+
rescue Interrupt
|
118
|
+
puts 'Interrupted'
|
119
|
+
stopped = true
|
145
120
|
end
|
146
|
-
|
147
121
|
end
|
148
122
|
end
|
149
123
|
|
@@ -1,9 +1,7 @@
|
|
1
|
-
require 'rubygems'
|
2
1
|
require 'yaml'
|
3
|
-
require 'amqp'
|
4
|
-
require 'eventmachine'
|
5
2
|
require 'json'
|
6
3
|
require 'logger'
|
4
|
+
require 'bunny'
|
7
5
|
|
8
6
|
require 'ghtorrent/api_client'
|
9
7
|
require 'ghtorrent/settings'
|
@@ -53,11 +51,10 @@ class GHTMirrorEvents < GHTorrent::Command
|
|
53
51
|
(new1, dupl1, stored1) = store_count events
|
54
52
|
stored = stored | stored1
|
55
53
|
new = new + new1
|
56
|
-
new
|
57
54
|
end
|
58
55
|
|
59
56
|
stored.each do |e|
|
60
|
-
key = "evt
|
57
|
+
key = "evt.#{e['type']}"
|
61
58
|
exchange.publish e['id'], :persistent => true, :routing_key => key
|
62
59
|
end
|
63
60
|
return new, dupl
|
@@ -71,46 +68,37 @@ class GHTMirrorEvents < GHTorrent::Command
|
|
71
68
|
@persister = connect(:mongo, @settings)
|
72
69
|
@logger = Logger.new(STDOUT)
|
73
70
|
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
79
|
-
Signal.trap('TERM') {
|
80
|
-
info "Received SIGTERM, exiting"
|
81
|
-
AMQP.stop { EM.stop }
|
82
|
-
}
|
71
|
+
conn = Bunny.new(:host => config(:amqp_host),
|
72
|
+
:port => config(:amqp_port),
|
73
|
+
:username => config(:amqp_username),
|
74
|
+
:password => config(:amqp_password))
|
75
|
+
conn.start
|
83
76
|
|
84
|
-
|
85
|
-
|
86
|
-
:port => config(:amqp_port),
|
87
|
-
:username => config(:amqp_username),
|
88
|
-
:password => config(:amqp_password)) do |connection|
|
77
|
+
ch = conn.create_channel
|
78
|
+
@logger.debug "Connection to #{config(:amqp_host)} succeded"
|
89
79
|
|
90
|
-
|
91
|
-
|
80
|
+
exchange = ch.topic(config(:amqp_exchange), :durable => true,
|
81
|
+
:auto_delete => false)
|
92
82
|
|
93
|
-
|
94
|
-
|
95
|
-
|
96
|
-
|
97
|
-
:auto_delete => false)
|
98
|
-
|
99
|
-
# Retrieve events
|
100
|
-
EventMachine.add_periodic_timer(5) do
|
83
|
+
dupl_msgs = new_msgs = loops = 0
|
84
|
+
stopped = false
|
85
|
+
while not stopped
|
86
|
+
begin
|
101
87
|
(new, dupl) = retrieve exchange
|
102
88
|
dupl_msgs += dupl
|
103
89
|
new_msgs += new
|
104
|
-
|
105
|
-
|
106
|
-
|
107
|
-
|
108
|
-
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
-
|
90
|
+
loops += 1
|
91
|
+
sleep(5)
|
92
|
+
|
93
|
+
if loops >= 12 # One minute
|
94
|
+
ratio = (dupl_msgs.to_f / (dupl_msgs + new_msgs).to_f)
|
95
|
+
info("Stats: #{new_msgs} new, #{dupl_msgs} duplicate, ratio: #{ratio}")
|
96
|
+
dupl_msgs = new_msgs = loops = 0
|
97
|
+
end
|
98
|
+
rescue Interrupt
|
99
|
+
stopped = true
|
100
|
+
rescue Exception => e
|
101
|
+
@logger.error e
|
114
102
|
end
|
115
103
|
end
|
116
104
|
end
|
@@ -3,6 +3,8 @@ require 'ghtorrent/settings'
|
|
3
3
|
require 'ghtorrent/logging'
|
4
4
|
require 'ghtorrent/command'
|
5
5
|
require 'ghtorrent/retriever'
|
6
|
+
require "bunny"
|
7
|
+
|
6
8
|
|
7
9
|
class GHTRetrieveRepos < GHTorrent::Command
|
8
10
|
|
@@ -67,7 +69,7 @@ Values in the config.yaml file set with the -c command are overriden.
|
|
67
69
|
retriever.stop
|
68
70
|
}
|
69
71
|
|
70
|
-
retriever.run
|
72
|
+
retriever.run(self)
|
71
73
|
exit
|
72
74
|
else
|
73
75
|
debug "Parent #{Process.pid} forked child #{pid}"
|
@@ -122,80 +124,45 @@ class GHTRepoRetriever
|
|
122
124
|
@config
|
123
125
|
end
|
124
126
|
|
125
|
-
def run
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
127
|
+
def run(command)
|
128
|
+
|
129
|
+
processor = Proc.new do |msg|
|
130
|
+
owner, repo = msg.split(/ /)
|
131
|
+
user_entry = ght.transaction { ght.ensure_user(owner, false, false) }
|
130
132
|
|
131
|
-
|
132
|
-
warn
|
133
|
-
|
133
|
+
if user_entry.nil?
|
134
|
+
warn("Cannot find user #{owner}")
|
135
|
+
next
|
134
136
|
end
|
135
137
|
|
136
|
-
|
137
|
-
channel.auto_recovery = true
|
138
|
-
channel.prefetch(1)
|
138
|
+
repo_entry = ght.transaction { ght.ensure_repo(owner, repo) }
|
139
139
|
|
140
|
-
|
141
|
-
warn
|
142
|
-
|
140
|
+
if repo_entry.nil?
|
141
|
+
warn("Cannot find repository #{owner}/#{repo}")
|
142
|
+
next
|
143
143
|
end
|
144
144
|
|
145
|
-
|
146
|
-
:auto_delete => false)
|
145
|
+
debug("Retrieving repo #{owner}/#{repo}")
|
147
146
|
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
owner,repo = msg.split(/ /)
|
152
|
-
user_entry = ght.transaction { ght.ensure_user(owner, false, false) }
|
147
|
+
def send_message(function, user, repo)
|
148
|
+
ght.send(function, user, repo, refresh = false)
|
149
|
+
end
|
153
150
|
|
154
|
-
|
155
|
-
|
156
|
-
headers.ack
|
157
|
-
next
|
158
|
-
end
|
151
|
+
functions = %w(ensure_commits ensure_forks ensure_pull_requests
|
152
|
+
ensure_issues ensure_project_members ensure_watchers ensure_labels)
|
159
153
|
|
160
|
-
|
154
|
+
functions.each do |x|
|
161
155
|
|
162
|
-
|
163
|
-
|
164
|
-
|
156
|
+
begin
|
157
|
+
send_message(x, owner, repo)
|
158
|
+
rescue Exception
|
159
|
+
warn("Error processing #{x} for #{owner}/#{repo}")
|
165
160
|
next
|
166
161
|
end
|
167
|
-
|
168
|
-
debug("Retrieving repo #{owner}/#{repo}")
|
169
|
-
def send_message(function, user, repo)
|
170
|
-
ght.send(function, user, repo, refresh = false)
|
171
|
-
end
|
172
|
-
|
173
|
-
functions = %w(ensure_commits ensure_forks ensure_pull_requests
|
174
|
-
ensure_issues ensure_project_members ensure_watchers ensure_labels)
|
175
|
-
|
176
|
-
functions.each do |x|
|
177
|
-
|
178
|
-
begin
|
179
|
-
send_message(x, owner, repo)
|
180
|
-
rescue Interrupt
|
181
|
-
stop
|
182
|
-
rescue Exception
|
183
|
-
warn("Error processing #{x} for #{owner}/#{repo}")
|
184
|
-
next
|
185
|
-
end
|
186
|
-
|
187
|
-
if @stop
|
188
|
-
break
|
189
|
-
end
|
190
|
-
end
|
191
|
-
|
192
|
-
headers.ack
|
193
|
-
debug("Finished processing #{owner}/#{repo}")
|
194
|
-
if @stop
|
195
|
-
connection.disconnect{AMQP.stop { EM.stop }}
|
196
|
-
end
|
197
162
|
end
|
198
163
|
end
|
164
|
+
|
165
|
+
command.queue_client(@queue, :before, processor)
|
199
166
|
end
|
200
167
|
|
201
168
|
def stop
|
data/lib/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: ghtorrent
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: '0.9'
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Georgios Gousios
|
@@ -9,24 +9,24 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2014-01-11 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
|
-
name:
|
15
|
+
name: mongo
|
16
16
|
requirement: !ruby/object:Gem::Requirement
|
17
17
|
requirements:
|
18
18
|
- - ~>
|
19
19
|
- !ruby/object:Gem::Version
|
20
|
-
version: 1.
|
20
|
+
version: 1.9.0
|
21
21
|
type: :runtime
|
22
22
|
prerelease: false
|
23
23
|
version_requirements: !ruby/object:Gem::Requirement
|
24
24
|
requirements:
|
25
25
|
- - ~>
|
26
26
|
- !ruby/object:Gem::Version
|
27
|
-
version: 1.
|
27
|
+
version: 1.9.0
|
28
28
|
- !ruby/object:Gem::Dependency
|
29
|
-
name:
|
29
|
+
name: bson_ext
|
30
30
|
requirement: !ruby/object:Gem::Requirement
|
31
31
|
requirements:
|
32
32
|
- - ~>
|
@@ -40,47 +40,47 @@ dependencies:
|
|
40
40
|
- !ruby/object:Gem::Version
|
41
41
|
version: 1.9.0
|
42
42
|
- !ruby/object:Gem::Dependency
|
43
|
-
name:
|
43
|
+
name: trollop
|
44
44
|
requirement: !ruby/object:Gem::Requirement
|
45
45
|
requirements:
|
46
46
|
- - ~>
|
47
47
|
- !ruby/object:Gem::Version
|
48
|
-
version:
|
48
|
+
version: 2.0.0
|
49
49
|
type: :runtime
|
50
50
|
prerelease: false
|
51
51
|
version_requirements: !ruby/object:Gem::Requirement
|
52
52
|
requirements:
|
53
53
|
- - ~>
|
54
54
|
- !ruby/object:Gem::Version
|
55
|
-
version:
|
55
|
+
version: 2.0.0
|
56
56
|
- !ruby/object:Gem::Dependency
|
57
|
-
name:
|
57
|
+
name: sequel
|
58
58
|
requirement: !ruby/object:Gem::Requirement
|
59
59
|
requirements:
|
60
60
|
- - ~>
|
61
61
|
- !ruby/object:Gem::Version
|
62
|
-
version:
|
62
|
+
version: 4.5.0
|
63
63
|
type: :runtime
|
64
64
|
prerelease: false
|
65
65
|
version_requirements: !ruby/object:Gem::Requirement
|
66
66
|
requirements:
|
67
67
|
- - ~>
|
68
68
|
- !ruby/object:Gem::Version
|
69
|
-
version:
|
69
|
+
version: 4.5.0
|
70
70
|
- !ruby/object:Gem::Dependency
|
71
|
-
name:
|
71
|
+
name: bunny
|
72
72
|
requirement: !ruby/object:Gem::Requirement
|
73
73
|
requirements:
|
74
74
|
- - ~>
|
75
75
|
- !ruby/object:Gem::Version
|
76
|
-
version:
|
76
|
+
version: 1.0.0
|
77
77
|
type: :runtime
|
78
78
|
prerelease: false
|
79
79
|
version_requirements: !ruby/object:Gem::Requirement
|
80
80
|
requirements:
|
81
81
|
- - ~>
|
82
82
|
- !ruby/object:Gem::Version
|
83
|
-
version:
|
83
|
+
version: 1.0.0
|
84
84
|
- !ruby/object:Gem::Dependency
|
85
85
|
name: rspec
|
86
86
|
requirement: !ruby/object:Gem::Requirement
|
@@ -109,9 +109,8 @@ dependencies:
|
|
109
109
|
- - ~>
|
110
110
|
- !ruby/object:Gem::Version
|
111
111
|
version: '1.16'
|
112
|
-
description:
|
113
|
-
|
114
|
-
to mirror and process Github data
|
112
|
+
description: ! "A library and a collection of associated programs\n to
|
113
|
+
mirror and process Github data"
|
115
114
|
email: gousiosg@gmail.com
|
116
115
|
executables:
|
117
116
|
- ght-data-retrieval
|
@@ -125,7 +124,6 @@ executables:
|
|
125
124
|
extensions: []
|
126
125
|
extra_rdoc_files: []
|
127
126
|
files:
|
128
|
-
- lib/ghtorrent.rb
|
129
127
|
- lib/ghtorrent/adapters/base_adapter.rb
|
130
128
|
- lib/ghtorrent/adapters/mongo_persister.rb
|
131
129
|
- lib/ghtorrent/adapters/noop_persister.rb
|
@@ -169,6 +167,7 @@ files:
|
|
169
167
|
- lib/ghtorrent/time.rb
|
170
168
|
- lib/ghtorrent/transacted_ghtorrent.rb
|
171
169
|
- lib/ghtorrent/utils.rb
|
170
|
+
- lib/ghtorrent.rb
|
172
171
|
- lib/version.rb
|
173
172
|
- bin/ght-data-retrieval
|
174
173
|
- bin/ght-get-more-commits
|
@@ -184,8 +183,8 @@ files:
|
|
184
183
|
- Gemfile
|
185
184
|
- Gemfile.lock
|
186
185
|
- LICENSE
|
187
|
-
- README.md
|
188
186
|
- Rakefile
|
187
|
+
- README.md
|
189
188
|
- spec/api_client_spec.rb
|
190
189
|
- spec/spec_helper.rb
|
191
190
|
homepage: https://github.com/gousiosg/github-mirror
|
@@ -198,17 +197,17 @@ require_paths:
|
|
198
197
|
- lib
|
199
198
|
required_ruby_version: !ruby/object:Gem::Requirement
|
200
199
|
requirements:
|
201
|
-
- - '>='
|
200
|
+
- - ! '>='
|
202
201
|
- !ruby/object:Gem::Version
|
203
202
|
version: '0'
|
204
203
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
205
204
|
requirements:
|
206
|
-
- - '>='
|
205
|
+
- - ! '>='
|
207
206
|
- !ruby/object:Gem::Version
|
208
207
|
version: '0'
|
209
208
|
requirements: []
|
210
209
|
rubyforge_project:
|
211
|
-
rubygems_version: 2.
|
210
|
+
rubygems_version: 2.1.3
|
212
211
|
signing_key:
|
213
212
|
specification_version: 4
|
214
213
|
summary: Mirror and process Github data
|