couchrest_changes 0.2.1 → 0.2.2

data/lib/couchrest/changes.rb

@@ -3,6 +3,7 @@ require 'fileutils'
 require 'pathname'
 
 require 'couchrest/changes/config'
+require 'couchrest/changes/database_proxy'
 require 'couchrest/changes/observer'
 
 module CouchRest

data/lib/couchrest/changes/database_proxy.rb

@@ -0,0 +1,41 @@
+module CouchRest::Changes
+
+  #
+  # CouchRest uses curl for 'streaming' requests
+  # (requests with a block passed to the db).
+  #
+  # Unfortunately, this leaks the username and password in the process list.
+  # We don't want to do this. So, we create two separate CouchRest::Database
+  # instances: one that is for normal requests and one that is used for
+  # streaming requests. The streaming one we hack to use netrc file in order
+  # to keep authentication info out of the process list.
+  #
+  # If no netrc file is configure, then this DatabaseProxy just uses the
+  # regular db.
+  #
+  class DatabaseProxy
+    def initialize(db_name)
+      @db = CouchRest.new(Config.couch_host).database(db_name)
+      unless @db
+        Config.logger.error { "Database #{db_name} not found!" }
+        raise RuntimeError "Database #{db_name} not found!"
+      end
+      if Config.connection[:netrc] && !Config.connection[:netrc].empty?
+        @db_stream = CouchRest.new(Config.couch_host_no_auth).database(db_name)
+        streamer = @db_stream.instance_variable_get('@streamer') # cheating, not exposed.
+        streamer.default_curl_opts += " --netrc-file \"#{Config.connection[:netrc]}\""
+      else
+        @db_stream = @db
+      end
+    end
+
+    def changes(*args, &block)
+      if block
+        @db_stream.changes(*args, &block)
+      else
+        @db.changes(*args)
+      end
+    end
+  end
+
+end

data/lib/couchrest/changes/observer.rb

@@ -1,43 +1,5 @@
 module CouchRest::Changes
 
-  #
-  # CouchRest uses curl for 'streaming' requests
-  # (requests with a block passed to the db).
-  #
-  # Unfortunately, this leaks the username and password in the process list.
-  # We don't want to do this. So, we create two separate CouchRest::Database
-  # instances: one that is for normal requests and one that is used for
-  # streaming requests. The streaming one we hack to use netrc file in order
-  # to keep authentication info out of the process list.
-  #
-  # If no netrc file is configure, then this DatabaseProxy just uses the
-  # regular db.
-  #
-  class DatabaseProxy
-    def initialize(db_name)
-      @db = CouchRest.new(Config.couch_host).database(db_name)
-      unless @db
-        Config.logger.error "Database #{db_name} not found!"
-        raise RuntimeError "Database #{db_name} not found!"
-      end
-      if Config.connection[:netrc] && !Config.connection[:netrc].empty?
-        @db_stream = CouchRest.new(Config.couch_host_no_auth).database(db_name)
-        streamer = @db_stream.instance_variable_get('@streamer') # cheating, not exposed.
-        streamer.default_curl_opts += " --netrc-file \"#{Config.connection[:netrc]}\""
-      else
-        @db_stream = @db
-      end
-    end
-
-    def changes(*args, &block)
-      if block
-        @db_stream.changes(*args, &block)
-      else
-        @db.changes(*args)
-      end
-    end
-  end
-
   #
   # NOTE: the sequence stored in the variable @since is a different format
   # depending on which flavor of couchdb is being used. For normal couchdb
@@ -51,13 +13,13 @@ module CouchRest::Changes
 
     def initialize(db_name, options = {})
       @db_name = Config.complete_db_name(db_name)
-      info "Tracking #{db_name}"
-      debug "Options: #{options.inspect}" if options.keys.any?
+      info { "Tracking #{db_name}" }
+      debug { "Options: #{options.inspect}" } if options.keys.any?
       @options = options
       @db = DatabaseProxy.new(@db_name)
       setup_sequence_file(@db_name)
       unless rerun?
-        @since = read_seq(@db_name)
+        @since = read_or_reset_sequence(@db_name)
       else
         @since = 0
       end
@@ -84,8 +46,7 @@ module CouchRest::Changes
     end
 
     def listen
-      info "listening..."
-      debug "Starting at sequence #{since}"
+      info { "Listening to #{@db_name}/_changes starting at sequence #{since}" }
       last = nil
       result = @db.changes(feed_options) do |hash|
         last = hash
@@ -97,9 +58,9 @@ module CouchRest::Changes
     # appearently MultiJson has issues with the end of the couch stream.
     # So sometimes we get a MultiJson::LoadError instead...
     rescue MultiJson::LoadError, EOFError, RestClient::ServerBrokeConnection => exc
-      error "Couch stream ended - #{exc.class}"
-      debug result.inspect if result
-      debug last.inspect if last
+      error { "Couch #{@db_name}/_changes stream ended - #{exc.class}" }
+      debug { result.inspect } if result
+      debug { last.inspect } if last
       retry if retry_without_sequence?(result, last) || retry_later?
     end
 
@@ -156,6 +117,25 @@ module CouchRest::Changes
       end
     end
 
+    #
+    # if the sequence in the database is newer than the sequence stored
+    # in the sequence file, then we need to reset the stored sequence
+    # to what is in the database.
+    #
+    def read_or_reset_sequence(db_name)
+      sequence = read_seq(db_name)
+      if sequence != 0
+        seq_number       = parse_sequence_number(sequence)
+        seq_number_in_db = parse_sequence_number(last_sequence)
+        if seq_number_in_db < seq_number
+          info { "Stored sequence (#{seq_number}) is greater than in db (#{seq_number_in_db}), resetting sequence to 0." }
+          sequence = 0
+          store_seq(db_name, sequence)
+        end
+      end
+      return sequence
+    end
+
     #
     # reads the sequence file, e.g. (/var/run/tapicero/users.seq), returning
     # the sequence number or zero if the sequence number could not be
@@ -163,17 +143,17 @@ module CouchRest::Changes
     #
     def read_seq(db_name)
       filename = sequence_file_name(db_name)
-      debug "Looking up sequence here: #{filename}"
+      debug { "Looking up sequence here: #{filename}" }
       result = File.read(filename)
       if result.empty?
-        debug "Found no sequence in the file #{filename}."
+        debug { "Found no sequence in the file #{filename}." }
         return 0
       else
-        debug "Found sequence: #{result}"
+        debug { "Found sequence: #{result}" }
         return result
       end
     rescue Errno::ENOENT => e
-      warn "No sequence file found. Starting from scratch (#{filename})"
+      warn { "No sequence file found. Starting from scratch (#{filename})" }
       return 0
     end
 
@@ -189,7 +169,7 @@ module CouchRest::Changes
     def retry_without_sequence?(result, last_hash)
       if malformated_sequence?(result) || malformated_sequence?(last_hash)
         @since = 0
-        info "Trying to start from scratch (db #{@db_name})."
+        info { "Trying to start from scratch (db #{@db_name})." }
         debug { {:result => result, :last_hash => last_hash}.inspect }
       end
     end
@@ -199,14 +179,29 @@ module CouchRest::Changes
       reason && ( reason.include?('since') || reason == 'badarg' )
     end
 
+    #
+    # Sequence might be a number, a number as a string, an array,
+    # or an array encoded as a string. This method returns the
+    # integer value of the number part of the sequence.
+    #
+    def parse_sequence_number(seq)
+      if seq.is_a? String
+        seq = MultiJson.decode(seq)
+      end
+      [seq].flatten.first.to_i
+    rescue Exception => exc
+      error { "Failed to parse sequence #{seq.inspect} (#{exc})." }
+      0
+    end
+
     def hash_for_change?(hash)
       hash["id"] && hash["changes"]
     end
 
     def retry_later?
       return unless rerun?
-      info "Will retry in 15 seconds."
-      info "Retried #{retry_count} times so far."
+      info { "Will retry in 15 seconds." }
+      info { "Retried #{retry_count} times so far." }
       sleep 15
       @retry_count += 1
     end

data/lib/couchrest/changes/version.rb

@@ -1,5 +1,5 @@
 module CouchRest
   module Changes
-    VERSION = "0.2.1"
+    VERSION = "0.2.2"
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: couchrest_changes
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-01 00:00:00.000000000 Z
+date: 2015-04-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: couchrest
@@ -133,6 +133,7 @@ extra_rdoc_files: []
 files:
 - lib/couchrest/changes.rb
 - lib/couchrest/changes/config.rb
+- lib/couchrest/changes/database_proxy.rb
 - lib/couchrest/changes/observer.rb
 - lib/couchrest/changes/version.rb
 - Rakefile