couchrest_changes 0.2.1 → 0.2.2
Sign up to get free protection for your applications and to get access to all the features.
data/lib/couchrest/changes.rb
CHANGED
@@ -0,0 +1,41 @@
|
|
1
|
+
module CouchRest::Changes
|
2
|
+
|
3
|
+
#
|
4
|
+
# CouchRest uses curl for 'streaming' requests
|
5
|
+
# (requests with a block passed to the db).
|
6
|
+
#
|
7
|
+
# Unfortunately, this leaks the username and password in the process list.
|
8
|
+
# We don't want to do this. So, we create two separate CouchRest::Database
|
9
|
+
# instances: one that is for normal requests and one that is used for
|
10
|
+
# streaming requests. The streaming one we hack to use netrc file in order
|
11
|
+
# to keep authentication info out of the process list.
|
12
|
+
#
|
13
|
+
# If no netrc file is configure, then this DatabaseProxy just uses the
|
14
|
+
# regular db.
|
15
|
+
#
|
16
|
+
class DatabaseProxy
|
17
|
+
def initialize(db_name)
|
18
|
+
@db = CouchRest.new(Config.couch_host).database(db_name)
|
19
|
+
unless @db
|
20
|
+
Config.logger.error { "Database #{db_name} not found!" }
|
21
|
+
raise RuntimeError "Database #{db_name} not found!"
|
22
|
+
end
|
23
|
+
if Config.connection[:netrc] && !Config.connection[:netrc].empty?
|
24
|
+
@db_stream = CouchRest.new(Config.couch_host_no_auth).database(db_name)
|
25
|
+
streamer = @db_stream.instance_variable_get('@streamer') # cheating, not exposed.
|
26
|
+
streamer.default_curl_opts += " --netrc-file \"#{Config.connection[:netrc]}\""
|
27
|
+
else
|
28
|
+
@db_stream = @db
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def changes(*args, &block)
|
33
|
+
if block
|
34
|
+
@db_stream.changes(*args, &block)
|
35
|
+
else
|
36
|
+
@db.changes(*args)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
end
|
@@ -1,43 +1,5 @@
|
|
1
1
|
module CouchRest::Changes
|
2
2
|
|
3
|
-
#
|
4
|
-
# CouchRest uses curl for 'streaming' requests
|
5
|
-
# (requests with a block passed to the db).
|
6
|
-
#
|
7
|
-
# Unfortunately, this leaks the username and password in the process list.
|
8
|
-
# We don't want to do this. So, we create two separate CouchRest::Database
|
9
|
-
# instances: one that is for normal requests and one that is used for
|
10
|
-
# streaming requests. The streaming one we hack to use netrc file in order
|
11
|
-
# to keep authentication info out of the process list.
|
12
|
-
#
|
13
|
-
# If no netrc file is configure, then this DatabaseProxy just uses the
|
14
|
-
# regular db.
|
15
|
-
#
|
16
|
-
class DatabaseProxy
|
17
|
-
def initialize(db_name)
|
18
|
-
@db = CouchRest.new(Config.couch_host).database(db_name)
|
19
|
-
unless @db
|
20
|
-
Config.logger.error "Database #{db_name} not found!"
|
21
|
-
raise RuntimeError "Database #{db_name} not found!"
|
22
|
-
end
|
23
|
-
if Config.connection[:netrc] && !Config.connection[:netrc].empty?
|
24
|
-
@db_stream = CouchRest.new(Config.couch_host_no_auth).database(db_name)
|
25
|
-
streamer = @db_stream.instance_variable_get('@streamer') # cheating, not exposed.
|
26
|
-
streamer.default_curl_opts += " --netrc-file \"#{Config.connection[:netrc]}\""
|
27
|
-
else
|
28
|
-
@db_stream = @db
|
29
|
-
end
|
30
|
-
end
|
31
|
-
|
32
|
-
def changes(*args, &block)
|
33
|
-
if block
|
34
|
-
@db_stream.changes(*args, &block)
|
35
|
-
else
|
36
|
-
@db.changes(*args)
|
37
|
-
end
|
38
|
-
end
|
39
|
-
end
|
40
|
-
|
41
3
|
#
|
42
4
|
# NOTE: the sequence stored in the variable @since is a different format
|
43
5
|
# depending on which flavor of couchdb is being used. For normal couchdb
|
@@ -51,13 +13,13 @@ module CouchRest::Changes
|
|
51
13
|
|
52
14
|
def initialize(db_name, options = {})
|
53
15
|
@db_name = Config.complete_db_name(db_name)
|
54
|
-
info "Tracking #{db_name}"
|
55
|
-
debug "Options: #{options.inspect}" if options.keys.any?
|
16
|
+
info { "Tracking #{db_name}" }
|
17
|
+
debug { "Options: #{options.inspect}" } if options.keys.any?
|
56
18
|
@options = options
|
57
19
|
@db = DatabaseProxy.new(@db_name)
|
58
20
|
setup_sequence_file(@db_name)
|
59
21
|
unless rerun?
|
60
|
-
@since =
|
22
|
+
@since = read_or_reset_sequence(@db_name)
|
61
23
|
else
|
62
24
|
@since = 0
|
63
25
|
end
|
@@ -84,8 +46,7 @@ module CouchRest::Changes
|
|
84
46
|
end
|
85
47
|
|
86
48
|
def listen
|
87
|
-
info "
|
88
|
-
debug "Starting at sequence #{since}"
|
49
|
+
info { "Listening to #{@db_name}/_changes starting at sequence #{since}" }
|
89
50
|
last = nil
|
90
51
|
result = @db.changes(feed_options) do |hash|
|
91
52
|
last = hash
|
@@ -97,9 +58,9 @@ module CouchRest::Changes
|
|
97
58
|
# appearently MultiJson has issues with the end of the couch stream.
|
98
59
|
# So sometimes we get a MultiJson::LoadError instead...
|
99
60
|
rescue MultiJson::LoadError, EOFError, RestClient::ServerBrokeConnection => exc
|
100
|
-
error "Couch stream ended - #{exc.class}"
|
101
|
-
debug result.inspect if result
|
102
|
-
debug last.inspect if last
|
61
|
+
error { "Couch #{@db_name}/_changes stream ended - #{exc.class}" }
|
62
|
+
debug { result.inspect } if result
|
63
|
+
debug { last.inspect } if last
|
103
64
|
retry if retry_without_sequence?(result, last) || retry_later?
|
104
65
|
end
|
105
66
|
|
@@ -156,6 +117,25 @@ module CouchRest::Changes
|
|
156
117
|
end
|
157
118
|
end
|
158
119
|
|
120
|
+
#
|
121
|
+
# if the sequence in the database is newer than the sequence stored
|
122
|
+
# in the sequence file, then we need to reset the stored sequence
|
123
|
+
# to what is in the database.
|
124
|
+
#
|
125
|
+
def read_or_reset_sequence(db_name)
|
126
|
+
sequence = read_seq(db_name)
|
127
|
+
if sequence != 0
|
128
|
+
seq_number = parse_sequence_number(sequence)
|
129
|
+
seq_number_in_db = parse_sequence_number(last_sequence)
|
130
|
+
if seq_number_in_db < seq_number
|
131
|
+
info { "Stored sequence (#{seq_number}) is greater than in db (#{seq_number_in_db}), resetting sequence to 0." }
|
132
|
+
sequence = 0
|
133
|
+
store_seq(db_name, sequence)
|
134
|
+
end
|
135
|
+
end
|
136
|
+
return sequence
|
137
|
+
end
|
138
|
+
|
159
139
|
#
|
160
140
|
# reads the sequence file, e.g. (/var/run/tapicero/users.seq), returning
|
161
141
|
# the sequence number or zero if the sequence number could not be
|
@@ -163,17 +143,17 @@ module CouchRest::Changes
|
|
163
143
|
#
|
164
144
|
def read_seq(db_name)
|
165
145
|
filename = sequence_file_name(db_name)
|
166
|
-
debug "Looking up sequence here: #{filename}"
|
146
|
+
debug { "Looking up sequence here: #{filename}" }
|
167
147
|
result = File.read(filename)
|
168
148
|
if result.empty?
|
169
|
-
debug "Found no sequence in the file #{filename}."
|
149
|
+
debug { "Found no sequence in the file #{filename}." }
|
170
150
|
return 0
|
171
151
|
else
|
172
|
-
debug "Found sequence: #{result}"
|
152
|
+
debug { "Found sequence: #{result}" }
|
173
153
|
return result
|
174
154
|
end
|
175
155
|
rescue Errno::ENOENT => e
|
176
|
-
warn "No sequence file found. Starting from scratch (#{filename})"
|
156
|
+
warn { "No sequence file found. Starting from scratch (#{filename})" }
|
177
157
|
return 0
|
178
158
|
end
|
179
159
|
|
@@ -189,7 +169,7 @@ module CouchRest::Changes
|
|
189
169
|
def retry_without_sequence?(result, last_hash)
|
190
170
|
if malformated_sequence?(result) || malformated_sequence?(last_hash)
|
191
171
|
@since = 0
|
192
|
-
info "Trying to start from scratch (db #{@db_name})."
|
172
|
+
info { "Trying to start from scratch (db #{@db_name})." }
|
193
173
|
debug { {:result => result, :last_hash => last_hash}.inspect }
|
194
174
|
end
|
195
175
|
end
|
@@ -199,14 +179,29 @@ module CouchRest::Changes
|
|
199
179
|
reason && ( reason.include?('since') || reason == 'badarg' )
|
200
180
|
end
|
201
181
|
|
182
|
+
#
|
183
|
+
# Sequence might be a number, a number as a string, an array,
|
184
|
+
# or an array encoded as a string. This method returns the
|
185
|
+
# integer value of the number part of the sequence.
|
186
|
+
#
|
187
|
+
def parse_sequence_number(seq)
|
188
|
+
if seq.is_a? String
|
189
|
+
seq = MultiJson.decode(seq)
|
190
|
+
end
|
191
|
+
[seq].flatten.first.to_i
|
192
|
+
rescue Exception => exc
|
193
|
+
error { "Failed to parse sequence #{seq.inspect} (#{exc})." }
|
194
|
+
0
|
195
|
+
end
|
196
|
+
|
202
197
|
def hash_for_change?(hash)
|
203
198
|
hash["id"] && hash["changes"]
|
204
199
|
end
|
205
200
|
|
206
201
|
def retry_later?
|
207
202
|
return unless rerun?
|
208
|
-
info "Will retry in 15 seconds."
|
209
|
-
info "Retried #{retry_count} times so far."
|
203
|
+
info { "Will retry in 15 seconds." }
|
204
|
+
info { "Retried #{retry_count} times so far." }
|
210
205
|
sleep 15
|
211
206
|
@retry_count += 1
|
212
207
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: couchrest_changes
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.2
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2015-04-
|
12
|
+
date: 2015-04-14 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: couchrest
|
@@ -133,6 +133,7 @@ extra_rdoc_files: []
|
|
133
133
|
files:
|
134
134
|
- lib/couchrest/changes.rb
|
135
135
|
- lib/couchrest/changes/config.rb
|
136
|
+
- lib/couchrest/changes/database_proxy.rb
|
136
137
|
- lib/couchrest/changes/observer.rb
|
137
138
|
- lib/couchrest/changes/version.rb
|
138
139
|
- Rakefile
|