couchrest_changes 0.0.5 → 0.1.0

data/lib/couchrest/changes.rb

@@ -3,174 +3,15 @@ require 'fileutils'
 require 'pathname'
 
 require 'couchrest/changes/config'
+require 'couchrest/changes/observer'
 
 module CouchRest
-  class Changes
+  module Changes
 
-    attr_writer :logger
-
-    def initialize(db_name, options = {})
-      db_name = Config.complete_db_name(db_name)
-      info "Tracking #{db_name}"
-      debug "Options: #{options.inspect}" if options.keys.any?
-      @options = options
-      @db = CouchRest.new(Config.couch_host).database(db_name)
-      read_seq(Config.seq_file) unless rerun?
-      check_seq
-    end
-
-    # triggered when a document was newly created
-    def created(hash = {}, &block)
-      run_or_define_hook :created, hash, &block
-    end
-
-    # triggered when a document was deleted
-    def deleted(hash = {}, &block)
-      run_or_define_hook :deleted, hash, &block
-    end
-
-    # triggered when an existing document was updated
-    def updated(hash = {}, &block)
-      run_or_define_hook :updated, hash, &block
-    end
-
-    # triggered whenever a document was changed
-    def changed(hash = {}, &block)
-      run_or_define_hook :changed, hash, &block
-    end
-
-    def listen
-      info "listening..."
-      debug "Starting at sequence #{since}"
-      result = db.changes feed_options do |hash|
-        @retry_count = 0
-        callbacks(hash)
-        store_seq(hash["seq"])
-      end
-      raise EOFError
-    # appearently MultiJson has issues with the end of the couch stream.
-    # So sometimes we get a MultiJson::LoadError instead...
-    rescue MultiJson::LoadError, EOFError, RestClient::ServerBrokeConnection
-      return if run_once?
-      log_and_recover(result)
-      retry
-    end
-
-    protected
-
-    def feed_options
-      if run_once?
-        { :since => since }
-      else
-        { :feed => :continuous, :since => since, :heartbeat => 1000 }
-      end.merge @options
-    end
-
-    def since
-      @since ||= 0  # fetch_last_seq
-    end
-
-    def callbacks(hash)
-      # let's not track design document changes
-      return if hash['id'].start_with? '_design/'
-      return unless changes = hash["changes"]
-      changed(hash)
-      return deleted(hash) if hash["deleted"]
-      return created(hash) if changes[0]["rev"].start_with?('1-')
-      updated(hash)
-    end
-
-    def run_or_define_hook(event, hash = {}, &block)
-      @callbacks ||= {}
-      if block_given?
-        @callbacks[event] = block
-      else
-        @callbacks[event] && @callbacks[event].call(hash)
-      end
-    end
-
-    def read_seq(filename)
-      debug "Looking up sequence here: #{filename}"
-      FileUtils.touch(filename)
-      unless File.writable?(filename)
-        raise StandardError.new("Can't write to sequence file #{filename}")
-      end
-      @since = File.read(filename)
-    rescue Errno::ENOENT => e
-      warn "No sequence file found. Starting from scratch"
-    end
-
-    def check_seq
-      if @since == ''
-        @since = nil
-        debug "Found no sequence in the file."
-      elsif @since
-        debug "Found sequence: #{@since}"
+    class << self
+      def new(*opts)
+        Observer.new(*opts)
       end
     end
-
-    def store_seq(seq)
-      File.write Config.seq_file, MultiJson.dump(seq)
-    end
-
-    def log_and_recover(result)
-      debug result.inspect if result
-      info "Couch stream ended unexpectedly."
-      info "Will retry in 15 seconds."
-      info "Retried #{retry_count} times so far."
-      sleep 15
-      @retry_count += 1
-    end
-
-    #
-    # UNUSED: this is useful for only following new sequences.
-    # might also require .to_json to work on bigcouch.
-    #
-    def fetch_last_seq
-      hash = db.changes :limit => 1, :descending => true
-      return hash["last_seq"]
-    end
-
-    def rerun?
-      Config.flags.include?('--rerun')
-    end
-
-    def run_once?
-      Config.flags.include?('--run-once')
-    end
-
-    def info(message)
-      return unless log_attempt?
-      logger.info message
-    end
-
-    def debug(message)
-      return unless log_attempt?
-      logger.debug message
-    end
-
-    def warn(message)
-      return unless log_attempt?
-      logger.warn message
-    end
-
-    # let's not clutter the logs if couch is down for a longer time.
-    def log_attempt?
-      [0, 1, 2, 4, 8, 20, 40, 120].include?(retry_count) ||
-        retry_count % 240 == 0
-    end
-
-    def retry_count
-      @retry_count ||= 0
-    end
-
-    def logger
-      logger ||= Config.logger
-    end
-
-    def db
-      @db
-    end
-
   end
 end

data/lib/couchrest/changes/config.rb

@@ -1,7 +1,7 @@
 require 'yaml'
 
 module CouchRest
-  class Changes
+  module Changes
     module Config
       extend self
 
@@ -20,6 +20,7 @@ module CouchRest
           file = find_file(file_path)
           load_config(file)
         end
+        self.flags ||= []
         init_logger
         log_loaded_configs(loaded.compact)
         logger.info "Observing #{couch_host_without_password}"

data/lib/couchrest/changes/observer.rb

@@ -0,0 +1,178 @@
+module CouchRest::Changes
+  class Observer
+
+    attr_writer :logger
+
+    def initialize(db_name, options = {})
+      db_name = Config.complete_db_name(db_name)
+      info "Tracking #{db_name}"
+      debug "Options: #{options.inspect}" if options.keys.any?
+      @options = options
+      unless @db = CouchRest.new(Config.couch_host).database(db_name)
+        logger.error "Database #{db_name} not found!"
+        raise RuntimeError "Database #{db_name} not found!"
+      end
+      read_seq(Config.seq_file) unless rerun?
+      check_seq
+    end
+
+    # triggered when a document was newly created
+    def created(hash = {}, &block)
+      run_or_define_hook :created, hash, &block
+    end
+
+    # triggered when a document was deleted
+    def deleted(hash = {}, &block)
+      run_or_define_hook :deleted, hash, &block
+    end
+
+    # triggered when an existing document was updated
+    def updated(hash = {}, &block)
+      run_or_define_hook :updated, hash, &block
+    end
+
+    # triggered whenever a document was changed
+    def changed(hash = {}, &block)
+      run_or_define_hook :changed, hash, &block
+    end
+
+    def listen
+      info "listening..."
+      debug "Starting at sequence #{since}"
+      result = db.changes feed_options do |hash|
+        @retry_count = 0
+        callbacks(hash)
+        store_seq(hash["seq"])
+      end
+      raise EOFError
+    # appearently MultiJson has issues with the end of the couch stream.
+    # So sometimes we get a MultiJson::LoadError instead...
+    rescue MultiJson::LoadError, EOFError, RestClient::ServerBrokeConnection
+      retry if retry_without_sequence?(result) || retry_later?
+      info "Couch stream ended."
+    end
+
+    def last_sequence
+      hash = db.changes :limit => 1, :descending => true
+      return hash["last_seq"]
+    end
+
+    protected
+
+    def feed_options
+      if run_once?
+        { :since => since }
+      else
+        { :feed => :continuous, :since => since, :heartbeat => 1000 }
+      end.merge @options
+    end
+
+    def since
+      @since ||= 0  # last_sequence
+    end
+
+    def callbacks(hash)
+      # let's not track design document changes
+      return if hash['id'].start_with? '_design/'
+      return unless changes = hash["changes"]
+      changed(hash)
+      return deleted(hash) if hash["deleted"]
+      return created(hash) if changes[0]["rev"].start_with?('1-')
+      updated(hash)
+    end
+
+    def run_or_define_hook(event, hash = {}, &block)
+      @callbacks ||= {}
+      if block_given?
+        @callbacks[event] = block
+      else
+        @callbacks[event] && @callbacks[event].call(hash)
+      end
+    end
+
+    def read_seq(filename)
+      debug "Looking up sequence here: #{filename}"
+      FileUtils.touch(filename)
+      unless File.writable?(filename)
+        raise StandardError.new("Can't write to sequence file #{filename}")
+      end
+      @since = File.read(filename)
+    rescue Errno::ENOENT => e
+      warn "No sequence file found. Starting from scratch"
+    end
+
+    def check_seq
+      if @since == ''
+        @since = nil
+        debug "Found no sequence in the file."
+      elsif @since
+        debug "Found sequence: #{@since}"
+      end
+    end
+
+    def store_seq(seq)
+      File.write Config.seq_file, MultiJson.dump(seq)
+    end
+
+    def retry_without_sequence?(result)
+      return unless malformated_sequence?(result)
+      @since = nil
+      info "Trying to start from scratch."
+    end
+
+    def malformated_sequence?(result)
+      reason = result && result.respond_to?(:keys) && result["reason"]
+      reason && ( reason.include?('since') || reason == 'badarg' )
+    end
+
+    def retry_later?
+      return unless rerun?
+      info "Will retry in 15 seconds."
+      info "Retried #{retry_count} times so far."
+      sleep 15
+      @retry_count += 1
+    end
+
+    def rerun?
+      Config.flags.include?('--rerun')
+    end
+
+    def run_once?
+      Config.flags.include?('--run-once')
+    end
+
+    def info(message)
+      return unless log_attempt?
+      logger.info message
+    end
+
+    def debug(message)
+      return unless log_attempt?
+      logger.debug message
+    end
+
+    def warn(message)
+      return unless log_attempt?
+      logger.warn message
+    end
+
+    # let's not clutter the logs if couch is down for a longer time.
+    def log_attempt?
+      [0, 1, 2, 4, 8, 20, 40, 120].include?(retry_count) ||
+        retry_count % 240 == 0
+    end
+
+    def retry_count
+      @retry_count ||= 0
+    end
+
+    def logger
+      logger ||= Config.logger
+    end
+
+    def db
+      @db
+    end
+
+  end
+end

data/lib/couchrest/changes/version.rb

@@ -1,5 +1,5 @@
 module CouchRest
-  class Changes
-    VERSION = "0.0.5"
+  module Changes
+    VERSION = "0.1.0"
   end
 end

data/test/config.yaml

@@ -0,0 +1,19 @@
+#
+# Default configuration options for Tapicero
+#
+
+# couch connection configuration
+connection:
+  protocol: "http"
+  host: "localhost"
+  port: 5984
+  username: anna
+  password: secret
+  prefix: "couchrest_changes_test"
+  suffix: ""
+
+# file to store the last processed user record in so we can resume after
+# a restart:
+seq_file: "/tmp/couchrest_changes_test.seq"
+log_file: "/tmp/couchrest_changes_test.log"
+log_level: debug

data/test/integration/callback_test.rb

@@ -0,0 +1,34 @@
+require 'test_helper'
+
+class CallbackTest < CouchRest::Changes::IntegrationTest
+
+  def setup
+    super
+    @config.flags = ['--run-once']
+    @changes = CouchRest::Changes.new 'records'
+    File.write config.seq_file, @changes.last_sequence
+  end
+
+  def teardown
+    delete
+  end
+
+  def test_triggers_created
+    handler = mock 'handler'
+    handler.expects(:callback).once
+    @changes.created { |hash| handler.callback(hash) }
+    create
+    @changes.listen
+  end
+
+  def test_starts_from_scratch_on_invalid_sequence
+    File.write config.seq_file, "invalid string"
+    @changes = CouchRest::Changes.new 'records'
+    handler = mock 'handler'
+    handler.expects(:callback).at_least_once
+    @changes.created { |hash| handler.callback(hash) }
+    create
+    @changes.listen
+  end
+
+end

data/test/setup_couch.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+HOST="http://localhost:5984"
+echo "couch version :"
+curl -X GET $HOST
+echo "creating unprivileged user :"
+curl -HContent-Type:application/json -XPUT $HOST/_users/org.couchdb.user:me --data-binary '{"_id": "org.couchdb.user:me","name": "me","roles": [],"type": "user","password": "pwd"}'
+echo "creating database to watch:"
+curl -X PUT $HOST/couchrest_changes_test_records
+echo "restricting database access :"
+curl -X PUT $HOST/couchrest_changes_test_records/_security -Hcontent-type:application/json --data-binary '{"admins":{"names":[],"roles":[]},"members":{"names":["me"],"roles":[]}}'
+echo "adding admin :"
+curl -X PUT $HOST/_config/admins/anna -d '"secret"'

data/test/support/integration_test.rb

@@ -0,0 +1,39 @@
+module CouchRest::Changes
+  class IntegrationTest < MiniTest::Unit::TestCase
+
+    attr_reader :config
+
+    def setup
+      @config ||= CouchRest::Changes::Config.load BASE_DIR,
+        'test/config.yaml'
+    end
+
+    def create(fast = false)
+      result = database.save_doc :some => :content
+      raise RuntimeError.new(result.inspect) unless result['ok']
+      @record = {'_id' => result["id"], '_rev' => result["rev"]}
+      sleep 1
+    end
+
+    def delete(fast = false)
+      return if @record.nil? or @record['_deleted']
+      result = database.delete_doc @record
+      raise RuntimeError.new(result.inspect) unless result['ok']
+      @record['_deleted'] = true
+      sleep 1
+    end
+
+    def database
+      @database ||= host.database(database_name)
+    end
+
+    def database_name
+      config.complete_db_name('records')
+    end
+
+    def host
+      @host ||= CouchRest.new(config.couch_host)
+    end
+
+  end
+end

data/test/test_helper.rb

@@ -4,5 +4,8 @@ require 'minitest/autorun'
 BASE_DIR = File.expand_path('../..', __FILE__)
 $:.unshift File.expand_path('lib', BASE_DIR)
 
+require 'couchrest/changes'
+require 'support/integration_test'
+
 require 'mocha/setup'
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: couchrest_changes
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.1.0
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-02 00:00:00.000000000 Z
+date: 2014-07-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: couchrest
@@ -133,9 +133,14 @@ extra_rdoc_files: []
 files:
 - lib/couchrest/changes/config.rb
 - lib/couchrest/changes/version.rb
+- lib/couchrest/changes/observer.rb
 - lib/couchrest/changes.rb
 - Rakefile
 - Readme.md
+- test/integration/callback_test.rb
+- test/setup_couch.sh
+- test/config.yaml
+- test/support/integration_test.rb
 - test/test_helper.rb
 homepage: https://leap.se
 licenses: []
@@ -157,10 +162,14 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.25
+rubygems_version: 1.8.28
 signing_key: 
 specification_version: 3
 summary: CouchRest::Changes - Observe a couch database for changes and react upon
   them
 test_files:
+- test/integration/callback_test.rb
+- test/setup_couch.sh
+- test/config.yaml
+- test/support/integration_test.rb
 - test/test_helper.rb