barnyard_harvester 0.0.5 → 0.0.7

data/.gitignore

@@ -3,4 +3,5 @@
 .bundle
 Gemfile.lock
 pkg/*
+*.DS_Store
 1-BarnyardHarvester::*

data/barnyard_harvester.gemspec

@@ -26,4 +26,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "uuid"
   gem.add_runtime_dependency "bson_ext", "= 1.6.0"
   gem.add_runtime_dependency "mongo", "= 1.6.0"
+  gem.add_runtime_dependency "amqp"
+  gem.add_runtime_dependency "aws-sdk"
+
 end

data/lib/barnyard_harvester.rb

@@ -4,6 +4,7 @@ require "logger"
 require "barnyard_harvester/version"
 
 module BarnyardHarvester
+
   ADD = "add"
   CHANGE = "change"
   DELETE = "delete"
@@ -24,6 +25,16 @@ module BarnyardHarvester
       @debug = args.fetch(:debug) { false }
       @log = args.fetch(:logger) { Logger.new(STDOUT) }
 
+      @queueing = args[:queueing]
+
+      case @queueing
+        when :rabbitmq
+          @rabbitmq_settings = args.fetch(:rabbitmq_settings) { raise "You must provide :rabbitmq_settings" }
+        when :sqs
+          @sqs_settings = args.fetch(:sqs_settings) { raise "You must provide :sqs_settings" }
+        else
+          @queueing = :resque
+      end
 
       @backend = args.fetch(:backend) { :redis }
 
@@ -32,7 +43,7 @@ module BarnyardHarvester
       end
 
       require "barnyard_harvester/#{@backend.to_s}_helper" if File.exist? "barnyard_harvester/#{@backend.to_s}_helper"
-      require "barnyard_harvester/#{@backend.to_s}_queue"
+      require "barnyard_harvester/#{@queueing.to_s}_queue"
       require "barnyard_harvester/#{@backend.to_s}"
 
 #      YAML::ENGINE.yamler = 'syck'
@@ -114,12 +125,12 @@ module BarnyardHarvester
         end
       else
         # We got add!
-        begin
+        #begin
           @my_add_queue.push(@harvester_uuid, crop_change_uuid, @crop_number, primary_key, BarnyardHarvester::ADD, value)
-        rescue Exception => e
-          @log.fatal "FATAL error pushing add #{primary_key} to queue. #{e}"
-          exit 1
-        end
+        #rescue Exception => e
+        #  @log.fatal "FATAL error pushing add #{primary_key} to queue. #{e}"
+        #  exit 1
+        #end
 
         @my_barn[primary_key] = value
         @add_count += 1
@@ -130,6 +141,15 @@ module BarnyardHarvester
       "(#{@add_count}) adds, (#{@delete_count}) deletes, (#{@change_count}) changes, (#{@source_count}) source records, (#{@cache_count}) cache records"
     end
 
+    #def log_run(harvester_uuid, crop_number, began_at, ended_at, source_count, change_count, add_count, delete_count)
+    #
+    #  #begin
+    #  #rescue Exception => e
+    #  #  @log.fatal "#{self.class} Fail in Resque.enqueue of HarvesterLogs. #{e.backtrace}"
+    #  #end
+    #
+    #end
+
     def run
 
       @began_at = Time.now
@@ -141,13 +161,15 @@ module BarnyardHarvester
 
       @ended_at = Time.now
 
+
+      @my_add_queue.log_run(@harvester_uuid, @crop_number, @began_at, @ended_at, @source_count, @change_count, @add_count, @delete_count)
+
       # Let Farmer know I'm done and to flush the updates
       @my_barn.flush
       @my_add_queue.flush
       @my_change_queue.flush
       @my_delete_queue.flush
 
-      @my_barn.log_run(@harvester_uuid, @crop_number, @began_at, @ended_at, @source_count, @change_count, @add_count, @delete_count)
 
     end
 

data/lib/barnyard_harvester/dynamodb.rb

@@ -0,0 +1,88 @@
+require "crack"
+require "json"
+require "aws-sdk"
+
+module BarnyardHarvester
+
+  DEFAULT_DYNAMODB_SETTINGS = {
+      :dynamo_db_endpoint => "dynamodb.us-west-1.amazonaws.com",
+      :access_key_id => ENV["AWS_ACCESS_KEY_ID"],
+      :secret_access_key => ENV["AWS_SECRET_ACCESS_KEY"]
+  }
+
+  class Barn
+
+    def initialize(args)
+
+      @crop_number = args.fetch(:crop_number) { raise "You must provide :crop_number" }
+      @dynamodb_settings = args.fetch(:dynamodb_settings) { DEFAULT_DYNAMODB_SETTINGS }
+      @debug = args.fetch(:debug) { false }
+      @log = args.fetch(:logger) { Logger.new(STDOUT) }
+
+      @db = AWS::DynamoDB.new(@dynamodb_settings)
+
+      table_name = "barnyard_crop_id-#{@crop_number}"
+
+      begin
+        @table = @db.tables.create(table_name, 10, 5)
+        sleep 1 while @table.status == :creating
+        @table.hash_key = [:id, :string]
+      rescue AWS::DynamoDB::Errors::ResourceInUseException
+        @table = @db.tables[table_name]
+        @table.hash_key = [:id, :string]
+      end
+
+    end
+
+    def delete(primary_key)
+      check_key primary_key
+
+      item = @table.items.where("id" => primary_key).first # Save the value
+      value = item.attributes['value']
+      item.delete # Delete the key
+      Crack::JSON.parse(value) # Return the object
+    end
+
+    def []= primary_key, object
+      check_key primary_key
+      check_object object
+      @table.items.create('id' => primary_key, 'value' => object.to_json)
+    end
+
+    def [] primary_key
+      check_key primary_key
+
+      Crack::JSON.parse(@table.items.where("id" => primary_key).first.attributes['value'])
+    end
+
+    def has_key?(primary_key)
+      check_key primary_key
+
+       @table.items.where("id" => primary_key).count == 1
+
+    end
+
+    def each
+
+      @table.items.each do |i|
+        yield i.attributes['id'], i.attributes['value']
+      end
+
+    end
+
+    def flush
+    end
+
+    private
+
+    def check_key(primary_key)
+      # Raise an exception here if the key must conform to a specific format
+      # Example: raise "key must be a string object" unless key.is_a? String
+    end
+
+    def check_object(object)
+      raise "#{object.class} must implement the to_json method" unless object.respond_to? :to_json
+    end
+  end
+
+end

data/lib/barnyard_harvester/mongodb.rb

@@ -9,10 +9,6 @@ require "barnyard_harvester/mongodb_helper"
 
 module BarnyardHarvester
 
-  class HarvesterLogs
-    @queue = :logs_harvester
-  end
-
   DEFAULT_MONGO_SETTINGS = {
       :host_list => "localhost:27017",
       :collection => "test_collection",
@@ -33,7 +29,7 @@ module BarnyardHarvester
       @mongodb_settings.fetch(:collection) { raise "You must provide :collection" }
 
       @redis_settings.delete(:db)
-      Resque.redis = Redis.new(@redis_settings)
+      #Resque.redis = Redis.new(@redis_settings)
 
       @mongodb_settings[:debug] = @debug
       @mongodb_settings[:logger] = @log

data/lib/barnyard_harvester/rabbitmq_queue.rb

@@ -0,0 +1,106 @@
+module BarnyardHarvester
+
+  require "bunny"
+
+  QUEUE_FARMER = "barnyard-farmer"
+  QUEUE_HARVESTER = "barnyard-harvests"
+  QUEUE_TRANSACTION = "barnyard-transactions"
+  QUEUE_CHANGE = "barnyard-changes"
+
+  class Queue
+
+    def enqueue(queue, queued_at, harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value, old_value)
+
+      payload = Hash.new
+      payload[:queued_at] = queued_at
+      payload[:harvester_uuid] = harvester_uuid
+      payload[:crop_change_uuid] = crop_change_uuid
+      payload[:crop_number] = crop_number
+      payload[:primary_key] = primary_key
+      payload[:transaction_type] = transaction_type
+      payload[:value] = value
+      payload[:old_value] = old_value
+
+      json_payload = payload
+
+      @direct_exchange.publish(json_payload, key: queue)
+      @direct_exchange.publish(json_payload, key: QUEUE_CHANGE)
+
+    end
+
+    def log_run(harvester_uuid, crop_number, began_at, ended_at, source_count, change_count, add_count, delete_count)
+
+      payload = Hash.new
+      payload[:time] = Time.now
+      payload[:harvester_uuid] = harvester_uuid
+      payload[:crop_number] = crop_number
+      payload[:began_at] = began_at
+      payload[:ended_at] = ended_at
+      payload[:source_count] = source_count
+      payload[:change_count] = change_count
+      payload[:add_count] = add_count
+      payload[:delete_count] = delete_count
+
+      @direct_exchange.publish(payload.to_json, key: QUEUE_HARVESTER)
+
+    end
+
+    def initialize(args)
+
+      @debug = args.fetch(:debug) { false }
+      @log = args.fetch(:logger) { Logger.new(STDOUT) }
+      @crop_number = args.fetch(:crop_number) { raise "You must provide :crop_number" }
+      @rabbitmq_settings = args.fetch(:rabbitmq_settings) { raise "You must provide :rabbitmq_settings" }
+
+      @bunny = Bunny.new(@rabbitmq_settings)
+      @bunny.start
+
+      @direct_exchange = @bunny.exchange('');
+
+    end
+
+    def push(harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value, old_value=Hash.new)
+      check_key primary_key
+
+      enqueue(QUEUE_FARMER, DateTime.now, harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value.to_json, old_value.to_json)
+
+      message = "RabbitQueue: #{QUEUE_FARMER}, Now: #{DateTime.now}, Harvester:#{harvester_uuid}, Change:#{crop_change_uuid} crop_number: #{crop_number}, key: #{primary_key}, transaction_type: #{transaction_type})"
+
+      if @log.level == Logger::DEBUG
+        message += ", value: #{value.to_json}, old_value: #{old_value.to_json}"
+        @log.debug message
+      end
+    end
+
+    # Flush any data if needed.
+    #
+    def flush
+    end
+
+    private
+
+    # Raise an exception here if the key must conform to a specific format
+    #
+    def check_key(primary_key)
+      # Example: raise "key must be a string object" unless key.is_a? String
+      primary_key
+    end
+
+  end
+
+  # AddQueue
+  #
+  class AddQueue < Queue
+  end
+
+  # ChangeQueue
+  #
+  class ChangeQueue < Queue
+  end
+
+  # DeleteQueue
+  #
+  class DeleteQueue < Queue
+  end
+
+end

data/lib/barnyard_harvester/redis.rb

@@ -5,10 +5,6 @@ require "resque"
 
 module BarnyardHarvester
 
-  class HarvesterLogs
-    @queue = :logs_harvester
-  end
-
   class Barn
 
     def initialize(args)
@@ -20,7 +16,7 @@ module BarnyardHarvester
 
       @redis_settings.delete(:db)
 
-      Resque.redis = Redis.new(@redis_settings)
+      #Resque.redis = Redis.new(@redis_settings)
 
       # This sets the database number for redis to store the cached data
       @redis_settings[:db] = args[:crop_number]
@@ -30,16 +26,6 @@ module BarnyardHarvester
 
     end
 
-    def log_run(harvester_uuid, crop_number, began_at, ended_at, source_count, change_count, add_count, delete_count)
-
-      begin
-        Resque.enqueue(HarvesterLogs, Time.now, harvester_uuid, crop_number, began_at, ended_at, source_count, change_count, add_count, delete_count)
-      rescue Exception => e
-        logger.fatal "#{self.class} Fail in Resque.enqueue of HarvesterLogs. #{e.backtrace}"
-      end
-
-    end
-
     def delete(primary_key)
       check_key primary_key
 

data/lib/barnyard_harvester/{mongodb_queue.rb → resque_queue.rb}

@@ -4,21 +4,37 @@ module BarnyardHarvester
     @queue = :logs_change
   end
 
+  class HarvesterLogs
+    @queue = :logs_harvester
+  end
+
+  class DeliveryLogs
+    @queue = :logs_delivery
+  end
+
+  class TransactionLogs
+    @queue = :logs_transaction
+  end
   class Queue
 
-    class ResqueQueue
+    class Enqueue
       def initialize(queue, queued_at, harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value, old_value)
         Resque.enqueue(queue, queued_at, harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value, old_value)
-        Resque.enqueue(ChangeLogs,queued_at, harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value, old_value)
+        Resque.enqueue(ChangeLogs, queued_at, harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value, old_value)
       end
     end
 
+    def log_run(harvester_uuid, crop_number, began_at, ended_at, source_count, change_count, add_count, delete_count)
+
+      Resque.enqueue(HarvesterLogs, harvester_uuid, crop_number, began_at, ended_at, source_count, change_count, add_count, delete_count)
+
+    end
+
     def initialize(args)
 
       @debug = args.fetch(:debug) { false }
       @log = args.fetch(:logger) { Logger.new(STDOUT) }
-
-      raise "arguments must contain :crop_number => some_integer" if args[:crop_number].nil?
+      @crop_number = args.fetch(:crop_number) { raise "You must provide :crop_number" }
 
       resque_class_name = "Distribute"
 
@@ -30,17 +46,18 @@ module BarnyardHarvester
       rescue
         # Set the queue name to this apol_harvester's id prefixed with a Q_
         #Object.const_set(resque_class_name, Class.new { @queue =  "Q_#{args[:crop_number]}"})
-        Object.const_set(resque_class_name, Class.new { @queue =  "Farmer"})
+        Object.const_set(resque_class_name, Class.new { @queue = "Farmer" })
       end
 
       @resque_queue = Object.const_get(resque_class_name)
 
     end
 
+
     def push(harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value, old_value=Hash.new)
       check_key primary_key
 
-      ResqueQueue.new(@resque_queue, DateTime.now, harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value.to_json, old_value.to_json)
+      Enqueue.new(@resque_queue, DateTime.now, harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value.to_json, old_value.to_json)
 
       message = "RedisQueue: #{@resque_queue}, Now: #{DateTime.now}, Harvester:#{harvester_uuid}, Change:#{crop_change_uuid} crop_number: #{crop_number}, key: #{primary_key}, transaction_type: #{transaction_type})"
 

data/lib/barnyard_harvester/sqs_queue.rb

@@ -0,0 +1,112 @@
+module BarnyardHarvester
+
+  require "aws-sdk"
+
+  QUEUE_FARMER = "barnyard-farmer"
+  QUEUE_HARVESTER = "barnyard-harvests"
+  QUEUE_TRANSACTION = "barnyard-transactions"
+  QUEUE_CHANGE = "barnyard-changes"
+
+  class Queue
+
+    def enqueue(queued_at, harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value, old_value)
+
+      payload = Hash.new
+      payload[:queued_at] = queued_at
+      payload[:harvester_uuid] = harvester_uuid
+      payload[:crop_change_uuid] = crop_change_uuid
+      payload[:crop_number] = crop_number
+      payload[:primary_key] = primary_key
+      payload[:transaction_type] = transaction_type
+      payload[:value] = value
+      payload[:old_value] = old_value
+
+      json_payload = payload.to_json
+
+      @log.debug "Starting send_message to farmer..."
+      @farmer_queue.send_message(json_payload)
+      @log.debug "Done send_message to farmer..."
+
+      @log.debug "Starting send_message to changes..."
+      @change_queue.send_message(json_payload)
+      @log.debug "Done send_message to changes..."
+
+    end
+
+    def log_run(harvester_uuid, crop_number, began_at, ended_at, source_count, change_count, add_count, delete_count)
+
+      payload = Hash.new
+      payload[:time] = Time.now
+      payload[:harvester_uuid] = harvester_uuid
+      payload[:crop_number] = crop_number
+      payload[:began_at] = began_at
+      payload[:ended_at] = ended_at
+      payload[:source_count] = source_count
+      payload[:change_count] = change_count
+      payload[:add_count] = add_count
+      payload[:delete_count] = delete_count
+
+      @harvester_queue.send_message(payload.to_json)
+
+    end
+
+    def initialize(args)
+
+      @debug = args.fetch(:debug) { false }
+      @log = args.fetch(:logger) { Logger.new(STDOUT) }
+      @crop_number = args.fetch(:crop_number) { raise "You must provide :crop_number" }
+      @sqs_settings = args.fetch(:sqs_settings) { raise "You must provide :sqs_settings" }
+
+      @sqs = AWS::SQS.new(@sqs_settings)
+
+      @farmer_queue = @sqs.queues.create(QUEUE_FARMER)
+      @harvester_queue = @sqs.queues.create(QUEUE_HARVESTER)
+      @change_queue = @sqs.queues.create(QUEUE_CHANGE)
+
+    end
+
+    def push(harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value, old_value=Hash.new)
+      check_key primary_key
+
+      enqueue(DateTime.now, harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value.to_json, old_value.to_json)
+
+      message = "SQS: #{QUEUE_FARMER}, Now: #{DateTime.now}, Harvester:#{harvester_uuid}, Change:#{crop_change_uuid} crop_number: #{crop_number}, key: #{primary_key}, transaction_type: #{transaction_type})"
+
+      if @log.level == Logger::DEBUG
+        message += ", value: #{value.to_json}, old_value: #{old_value.to_json}"
+        @log.debug message
+      end
+    end
+
+    # Flush any data if needed.
+    #
+    def flush
+    end
+
+    private
+
+    # Raise an exception here if the key must conform to a specific format
+    #
+    def check_key(primary_key)
+      # Example: raise "key must be a string object" unless key.is_a? String
+      primary_key
+    end
+
+  end
+
+  # AddQueue
+  #
+  class AddQueue < Queue
+  end
+
+  # ChangeQueue
+  #
+  class ChangeQueue < Queue
+  end
+
+  # DeleteQueue
+  #
+  class DeleteQueue < Queue
+  end
+
+end

data/lib/barnyard_harvester/version.rb

@@ -1,3 +1,3 @@
 module BarnyardHarvester
-  VERSION = "0.0.5"
+  VERSION = "0.0.7"
 end

data/spec/dynamodb_sqs_spec.rb

@@ -0,0 +1,180 @@
+require "barnyard_harvester"
+require "yaml"
+require "logger"
+require "json"
+require "aws-sdk"
+
+CROP_NUMBER = 1
+
+DEFAULT_DYNAMODB_SETTINGS = {
+    :dynamo_db_endpoint => "dynamodb.us-west-1.amazonaws.com",
+    :access_key_id => ENV["AWS_ACCESS_KEY_ID"],
+    :secret_access_key => ENV["AWS_SECRET_ACCESS_KEY"]
+}
+
+SQS_SETTINGS = {
+    :sqs_endpoint => "sqs.us-west-1.amazonaws.com",
+    :access_key_id => ENV["AWS_ACCESS_KEY_ID"],
+    :secret_access_key => ENV["AWS_SECRET_ACCESS_KEY"]
+}
+
+describe BarnyardHarvester do
+
+  def load_and_process_file(file, backend)
+
+    data = YAML::load_file file
+
+    my_logger = Logger.new(STDOUT)
+    my_logger.level = Logger::DEBUG
+
+    h = BarnyardHarvester::Sync.new(:queueing => :sqs,
+                                    :sqs_settings => SQS_SETTINGS,
+                                    :backend => backend,
+                                    :debug => true,
+                                    :crop_number => CROP_NUMBER,
+                                    :dynamodb_settings => DEFAULT_DYNAMODB_SETTINGS,
+                                    :logger => my_logger)
+
+    h.run do
+      data.each do |primary_key, value|
+        h.process primary_key, value
+      end
+    end
+
+    h
+  end
+
+  def get_table
+    table_name = "barnyard_crop_id-#{CROP_NUMBER}"
+    db = AWS::DynamoDB.new(DEFAULT_DYNAMODB_SETTINGS)
+
+    begin
+      table = db.tables.create(table_name, 10, 5)
+      sleep 1 while table.status == :creating
+      table.hash_key = [:id, :string]
+      puts table.status
+    rescue AWS::DynamoDB::Errors::ResourceInUseException
+      table = db.tables[table_name]
+      table.hash_key = [:id, :string]
+    end
+    table
+  end
+
+  def flush
+
+    get_table.items.each do |i|
+      i.delete
+    end
+
+  end
+
+  before(:each) do
+
+    flush
+
+    @crop_number = 1
+
+    file = "spec/fixtures/data-init.yml"
+
+    data = YAML::load_file file
+
+    h = load_and_process_file(file, :dynamodb)
+
+    h.add_count.should eq(data.count)
+    h.delete_count.should eq(0)
+    h.change_count.should eq(0)
+    h.source_count.should eq(data.count)
+    h.cache_count.should eq(data.count)
+
+  end
+
+  it "test initial load of records" do
+
+    data = YAML::load_file "spec/fixtures/data-init.yml"
+
+    data.each do |primary_key, value|
+      value.to_json.should eq(get_table.items.where("id" => primary_key).first.attributes['value'])
+    end
+
+  end
+
+  it "test add one record" do
+
+    file = "spec/fixtures/data-add.yml"
+    data = YAML::load_file file
+
+    h = load_and_process_file(file, :dynamodb)
+
+    h.add_count.should eq(1)
+    h.delete_count.should eq(0)
+    h.change_count.should eq(0)
+    h.source_count.should eq(data.count)
+    h.cache_count.should eq(data.count)
+
+#    h.log_run("#{file}-#{Random.rand(100)}", @crop_number, Time.now, Time.now, h.source_count, h.change_count, h.add_count, h.delete_count)
+
+  end
+
+  it "test change one record" do
+
+    file = "spec/fixtures/data-change.yml"
+
+    data = YAML::load_file file
+
+    h = load_and_process_file(file, :dynamodb)
+
+    h.add_count.should eq(0)
+    h.delete_count.should eq(0)
+    h.change_count.should eq(1)
+    h.source_count.should eq(data.count)
+    h.cache_count.should eq(data.count)
+
+#    h.my_barn.log_run("#{file}-#{Random.rand(100)}", @crop_number, Time.now, Time.now, h.source_count, h.change_count, h.add_count, h.delete_count)
+
+  end
+
+  it "test delete one record" do
+
+    file = "spec/fixtures/data-delete.yml"
+
+    data = YAML::load_file file
+
+    h = load_and_process_file(file, :dynamodb)
+
+
+    h.add_count.should eq(0)
+    h.delete_count.should eq(1)
+    h.change_count.should eq(0)
+    h.source_count.should eq(data.count)
+    h.cache_count.should eq(data.count + 1)
+
+#    h.my_barn.log_run("#{file}-#{Random.rand(100)}", @crop_number, Time.now, Time.now, h.source_count, h.change_count, h.add_count, h.delete_count)
+
+  end
+
+  it "test delete all records and add one" do
+
+    init_file = "spec/fixtures/data-init.yml"
+    init_data = YAML::load_file init_file
+
+    file = "spec/fixtures/data-delete-all-records-add-one.yml"
+    #data = YAML::load_file file
+
+    h = load_and_process_file(file, :dynamodb)
+
+    h.add_count.should eq(1)
+    h.delete_count.should eq(5)
+    h.change_count.should eq(0)
+    h.source_count.should eq(1)
+    h.cache_count.should eq(init_data.count + 1)
+
+#    h.my_barn.log_run("#{file}-#{Random.rand(100)}", @crop_number, Time.now, Time.now, h.source_count, h.change_count, h.add_count, h.delete_count)
+
+  end
+
+
+  after(:each) do
+  end
+
+
+end

data/spec/hash_spec.rb

@@ -22,7 +22,12 @@ describe BarnyardHarvester do
     my_logger = Logger.new(STDOUT)
     my_logger.level = Logger::INFO
 
-    h = BarnyardHarvester::Sync.new(:backend => backend, :debug => false, :crop_number => 1, :hash_settings => redis_settings, :logger => my_logger)
+    h = BarnyardHarvester::Sync.new(:queueing => :resque,
+                                    :backend => backend,
+                                    :debug => false,
+                                    :crop_number => 1,
+                                    :redis_settings => redis_settings,
+                                    :logger => my_logger)
 
     h.run do
       data.each do |primary_key, value|

data/spec/mongo_spec.rb

@@ -29,7 +29,7 @@ MONGODB_REPLICA_SET_SETTINGS = {
     :collection => "test_collection"
 }
 
-$mongo_settings = MONGODB_REPLICA_SET_SETTINGS
+$mongo_settings = MONGODB_SETTINGS
 
 describe BarnyardHarvester do
 
@@ -40,7 +40,13 @@ describe BarnyardHarvester do
     my_logger = Logger.new(STDOUT)
     my_logger.level = Logger::INFO
 
-    h = BarnyardHarvester::Sync.new(:backend => backend, :debug => false, :mongodb_settings => $mongo_settings, :crop_number => CROP_NUMBER, :redis_settings => REDIS_SETTINGS, :logger => my_logger)
+    h = BarnyardHarvester::Sync.new(:backend => backend,
+                                    :queueing => :resque,
+                                    :debug => false,
+                                    :mongodb_settings => $mongo_settings,
+                                    :crop_number => CROP_NUMBER,
+                                    :redis_settings => REDIS_SETTINGS,
+                                    :logger => my_logger)
 
     h.run do
       data.each do |primary_key, value|
@@ -119,6 +125,8 @@ describe BarnyardHarvester do
     h.source_count.should eq(data.count)
     h.cache_count.should eq(data.count)
 
+    h.my_barn.log_run("#{file}-#{Random.rand(100)}", @crop_number, Time.now, Time.now, h.source_count, h.change_count, h.add_count, h.delete_count)
+
   end
 
   it "test change one record" do
@@ -135,6 +143,8 @@ describe BarnyardHarvester do
     h.source_count.should eq(data.count)
     h.cache_count.should eq(data.count)
 
+    h.my_barn.log_run("#{file}-#{Random.rand(100)}", @crop_number, Time.now, Time.now, h.source_count, h.change_count, h.add_count, h.delete_count)
+
   end
 
   it "test delete one record" do
@@ -145,13 +155,14 @@ describe BarnyardHarvester do
 
     h = load_and_process_file(file, :mongodb)
 
-
     h.add_count.should eq(0)
     h.delete_count.should eq(1)
     h.change_count.should eq(0)
     h.source_count.should eq(data.count)
     h.cache_count.should eq(data.count + 1)
 
+    h.my_barn.log_run("#{file}-#{Random.rand(100)}", @crop_number, Time.now, Time.now, h.source_count, h.change_count, h.add_count, h.delete_count)
+
   end
 
   it "test delete all records and add one" do
@@ -170,6 +181,8 @@ describe BarnyardHarvester do
     h.source_count.should eq(1)
     h.cache_count.should eq(init_data.count + 1)
 
+    h.my_barn.log_run("#{file}-#{Random.rand(100)}", @crop_number, Time.now, Time.now, h.source_count, h.change_count, h.add_count, h.delete_count)
+
   end
 
 

data/spec/redis_rabbitmq_spec.rb

@@ -0,0 +1,162 @@
+require "barnyard_harvester"
+require "yaml"
+require "redis"
+require "logger"
+require "json"
+
+CROP_NUMBER = 1
+
+REDIS_SETTINGS = {
+    :host => "localhost",
+    :port => 6379,
+    :db => CROP_NUMBER
+}
+
+RABBITMQ_SETTINGS = {
+    :host => "localhost"
+#    :port => 6163
+}
+
+describe BarnyardHarvester do
+
+  def load_and_process_file(file, backend)
+
+    data = YAML::load_file file
+
+    my_logger = Logger.new(STDOUT)
+    my_logger.level = Logger::INFO
+
+    h = BarnyardHarvester::Sync.new(:queueing => :rabbitmq, :rabbitmq_settings => RABBITMQ_SETTINGS, :backend => backend, :debug => false, :crop_number => CROP_NUMBER, :redis_settings => REDIS_SETTINGS, :logger => my_logger)
+
+    h.run do
+      data.each do |primary_key, value|
+        h.process primary_key, value
+      end
+    end
+
+    h
+  end
+
+  def flush
+
+    r = Redis.new(REDIS_SETTINGS)
+
+    r.keys.each do |k|
+      r.del k
+      #puts "deleted #{k}"
+    end
+
+  end
+
+  before(:each) do
+
+    flush
+
+    @crop_number = 1
+
+    file = "spec/fixtures/data-init.yml"
+
+    data = YAML::load_file file
+
+    h = load_and_process_file(file, :redis)
+
+    h.add_count.should eq(data.count)
+    h.delete_count.should eq(0)
+    h.change_count.should eq(0)
+    h.source_count.should eq(data.count)
+    h.cache_count.should eq(data.count)
+
+  end
+
+  it "test initial load of records" do
+
+    data = YAML::load_file "spec/fixtures/data-init.yml"
+
+    redis = Redis.new(REDIS_SETTINGS)
+
+    data.each do |primary_key, value|
+      value.to_json.should eq(redis.get(primary_key))
+    end
+
+  end
+
+  it "test add one record" do
+
+    file = "spec/fixtures/data-add.yml"
+    data = YAML::load_file file
+
+    h = load_and_process_file(file, :redis)
+
+    h.add_count.should eq(1)
+    h.delete_count.should eq(0)
+    h.change_count.should eq(0)
+    h.source_count.should eq(data.count)
+    h.cache_count.should eq(data.count)
+
+#    h.log_run("#{file}-#{Random.rand(100)}", @crop_number, Time.now, Time.now, h.source_count, h.change_count, h.add_count, h.delete_count)
+
+  end
+
+  it "test change one record" do
+
+    file = "spec/fixtures/data-change.yml"
+
+    data = YAML::load_file file
+
+    h = load_and_process_file(file, :redis)
+
+    h.add_count.should eq(0)
+    h.delete_count.should eq(0)
+    h.change_count.should eq(1)
+    h.source_count.should eq(data.count)
+    h.cache_count.should eq(data.count)
+
+#    h.my_barn.log_run("#{file}-#{Random.rand(100)}", @crop_number, Time.now, Time.now, h.source_count, h.change_count, h.add_count, h.delete_count)
+
+  end
+
+  it "test delete one record" do
+
+    file = "spec/fixtures/data-delete.yml"
+
+    data = YAML::load_file file
+
+    h = load_and_process_file(file, :redis)
+
+
+    h.add_count.should eq(0)
+    h.delete_count.should eq(1)
+    h.change_count.should eq(0)
+    h.source_count.should eq(data.count)
+    h.cache_count.should eq(data.count + 1)
+
+#    h.my_barn.log_run("#{file}-#{Random.rand(100)}", @crop_number, Time.now, Time.now, h.source_count, h.change_count, h.add_count, h.delete_count)
+
+  end
+
+  it "test delete all records and add one" do
+
+    init_file = "spec/fixtures/data-init.yml"
+    init_data = YAML::load_file init_file
+
+    file = "spec/fixtures/data-delete-all-records-add-one.yml"
+    #data = YAML::load_file file
+
+    h = load_and_process_file(file, :redis)
+
+    h.add_count.should eq(1)
+    h.delete_count.should eq(5)
+    h.change_count.should eq(0)
+    h.source_count.should eq(1)
+    h.cache_count.should eq(init_data.count + 1)
+
+#    h.my_barn.log_run("#{file}-#{Random.rand(100)}", @crop_number, Time.now, Time.now, h.source_count, h.change_count, h.add_count, h.delete_count)
+
+  end
+
+
+  after(:each) do
+  end
+
+
+end

data/spec/redis_sqs_spec.rb

@@ -0,0 +1,170 @@
+require "barnyard_harvester"
+require "yaml"
+require "redis"
+require "logger"
+require "json"
+
+CROP_NUMBER = 1
+
+REDIS_SETTINGS = {
+    :host => "localhost",
+    :port => 6379,
+    :db => CROP_NUMBER
+}
+
+
+SQS_SETTINGS = {
+    :sqs_endpoint => "sqs.us-west-1.amazonaws.com",
+    :access_key_id => ENV["AWS_ACCESS_KEY_ID"],
+    :secret_access_key => ENV["AWS_SECRET_ACCESS_KEY"]
+}
+
+describe BarnyardHarvester do
+
+  def load_and_process_file(file, backend)
+
+    data = YAML::load_file file
+
+    my_logger = Logger.new(STDOUT)
+    my_logger.level = Logger::DEBUG
+
+    h = BarnyardHarvester::Sync.new(:queueing => :sqs,
+                                    :sqs_settings => SQS_SETTINGS,
+                                    :backend => backend,
+                                    :debug => true,
+                                    :crop_number => CROP_NUMBER,
+                                    :redis_settings => REDIS_SETTINGS,
+                                    :logger => my_logger)
+
+    h.run do
+      data.each do |primary_key, value|
+        h.process primary_key, value
+      end
+    end
+
+    h
+  end
+
+  def flush
+
+    r = Redis.new(REDIS_SETTINGS)
+
+    r.keys.each do |k|
+      r.del k
+      #puts "deleted #{k}"
+    end
+
+  end
+
+  before(:each) do
+
+    flush
+
+    @crop_number = 1
+
+    file = "spec/fixtures/data-init.yml"
+
+    data = YAML::load_file file
+
+    h = load_and_process_file(file, :redis)
+
+    h.add_count.should eq(data.count)
+    h.delete_count.should eq(0)
+    h.change_count.should eq(0)
+    h.source_count.should eq(data.count)
+    h.cache_count.should eq(data.count)
+
+  end
+
+  it "test initial load of records" do
+
+    data = YAML::load_file "spec/fixtures/data-init.yml"
+
+    redis = Redis.new(REDIS_SETTINGS)
+
+    data.each do |primary_key, value|
+      value.to_json.should eq(redis.get(primary_key))
+    end
+
+  end
+
+  it "test add one record" do
+
+    file = "spec/fixtures/data-add.yml"
+    data = YAML::load_file file
+
+    h = load_and_process_file(file, :redis)
+
+    h.add_count.should eq(1)
+    h.delete_count.should eq(0)
+    h.change_count.should eq(0)
+    h.source_count.should eq(data.count)
+    h.cache_count.should eq(data.count)
+
+#    h.log_run("#{file}-#{Random.rand(100)}", @crop_number, Time.now, Time.now, h.source_count, h.change_count, h.add_count, h.delete_count)
+
+  end
+
+  it "test change one record" do
+
+    file = "spec/fixtures/data-change.yml"
+
+    data = YAML::load_file file
+
+    h = load_and_process_file(file, :redis)
+
+    h.add_count.should eq(0)
+    h.delete_count.should eq(0)
+    h.change_count.should eq(1)
+    h.source_count.should eq(data.count)
+    h.cache_count.should eq(data.count)
+
+#    h.my_barn.log_run("#{file}-#{Random.rand(100)}", @crop_number, Time.now, Time.now, h.source_count, h.change_count, h.add_count, h.delete_count)
+
+  end
+
+  it "test delete one record" do
+
+    file = "spec/fixtures/data-delete.yml"
+
+    data = YAML::load_file file
+
+    h = load_and_process_file(file, :redis)
+
+
+    h.add_count.should eq(0)
+    h.delete_count.should eq(1)
+    h.change_count.should eq(0)
+    h.source_count.should eq(data.count)
+    h.cache_count.should eq(data.count + 1)
+
+#    h.my_barn.log_run("#{file}-#{Random.rand(100)}", @crop_number, Time.now, Time.now, h.source_count, h.change_count, h.add_count, h.delete_count)
+
+  end
+
+  it "test delete all records and add one" do
+
+    init_file = "spec/fixtures/data-init.yml"
+    init_data = YAML::load_file init_file
+
+    file = "spec/fixtures/data-delete-all-records-add-one.yml"
+    #data = YAML::load_file file
+
+    h = load_and_process_file(file, :redis)
+
+    h.add_count.should eq(1)
+    h.delete_count.should eq(5)
+    h.change_count.should eq(0)
+    h.source_count.should eq(1)
+    h.cache_count.should eq(init_data.count + 1)
+
+#    h.my_barn.log_run("#{file}-#{Random.rand(100)}", @crop_number, Time.now, Time.now, h.source_count, h.change_count, h.add_count, h.delete_count)
+
+  end
+
+
+  after(:each) do
+  end
+
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: barnyard_harvester
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.7
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-11-13 00:00:00.000000000 Z
+date: 2012-11-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70118633756900 !ruby/object:Gem::Requirement
+  requirement: &70277157472920 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70118633756900
+  version_requirements: *70277157472920
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70118633753220 !ruby/object:Gem::Requirement
+  requirement: &70277157468160 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70118633753220
+  version_requirements: *70277157468160
 - !ruby/object:Gem::Dependency
   name: crack
-  requirement: &70118633748400 !ruby/object:Gem::Requirement
+  requirement: &70277157454620 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70118633748400
+  version_requirements: *70277157454620
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70118633735240 !ruby/object:Gem::Requirement
+  requirement: &70277157449620 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70118633735240
+  version_requirements: *70277157449620
 - !ruby/object:Gem::Dependency
   name: uuid
-  requirement: &70118633729920 !ruby/object:Gem::Requirement
+  requirement: &70277157443200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70118633729920
+  version_requirements: *70277157443200
 - !ruby/object:Gem::Dependency
   name: bson_ext
-  requirement: &70118633723020 !ruby/object:Gem::Requirement
+  requirement: &70277157439700 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - =
@@ -76,10 +76,10 @@ dependencies:
         version: 1.6.0
   type: :runtime
   prerelease: false
-  version_requirements: *70118633723020
+  version_requirements: *70277157439700
 - !ruby/object:Gem::Dependency
   name: mongo
-  requirement: &70118633718980 !ruby/object:Gem::Requirement
+  requirement: &70277157436860 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - =
@@ -87,7 +87,29 @@ dependencies:
         version: 1.6.0
   type: :runtime
   prerelease: false
-  version_requirements: *70118633718980
+  version_requirements: *70277157436860
+- !ruby/object:Gem::Dependency
+  name: amqp
+  requirement: &70277157435420 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70277157435420
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: &70277157431160 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70277157431160
 description: Performs harvests on data sources and detects adds, changes and deletes.
 email:
 - supercoder@gmail.com
@@ -105,15 +127,18 @@ files:
 - barnyard_harvester.gemspec
 - lib/.DS_Store
 - lib/barnyard_harvester.rb
+- lib/barnyard_harvester/dynamodb.rb
 - lib/barnyard_harvester/hash.rb
 - lib/barnyard_harvester/hash_queue.rb
 - lib/barnyard_harvester/mongodb.rb
 - lib/barnyard_harvester/mongodb_helper.rb
-- lib/barnyard_harvester/mongodb_queue.rb
+- lib/barnyard_harvester/rabbitmq_queue.rb
 - lib/barnyard_harvester/redis.rb
-- lib/barnyard_harvester/redis_queue.rb
+- lib/barnyard_harvester/resque_queue.rb
+- lib/barnyard_harvester/sqs_queue.rb
 - lib/barnyard_harvester/version.rb
 - lib/test.yml
+- spec/dynamodb_sqs_spec.rb
 - spec/fixtures/data-add.yml
 - spec/fixtures/data-change.yml
 - spec/fixtures/data-delete-all-records-add-one.yml
@@ -123,7 +148,9 @@ files:
 - spec/loader_spec.rb
 - spec/mongo_helper_spec.rb
 - spec/mongo_spec.rb
+- spec/redis_rabbitmq_spec.rb
 - spec/redis_spec.rb
+- spec/redis_sqs_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/jongillies/barnyard/tree/master/barnyard_harvester
 licenses: []
@@ -150,6 +177,7 @@ signing_key:
 specification_version: 3
 summary: Please check the README.md for more information.
 test_files:
+- spec/dynamodb_sqs_spec.rb
 - spec/fixtures/data-add.yml
 - spec/fixtures/data-change.yml
 - spec/fixtures/data-delete-all-records-add-one.yml
@@ -159,5 +187,7 @@ test_files:
 - spec/loader_spec.rb
 - spec/mongo_helper_spec.rb
 - spec/mongo_spec.rb
+- spec/redis_rabbitmq_spec.rb
 - spec/redis_spec.rb
+- spec/redis_sqs_spec.rb
 - spec/spec_helper.rb

data/lib/barnyard_harvester/redis_queue.rb

@@ -1,84 +0,0 @@
-module BarnyardHarvester
-
-  class ChangeLogs
-    @queue = :logs_change
-  end
-
-  class Queue
-
-    class ResqueQueue
-      def initialize(queue, queued_at, harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value, old_value)
-        Resque.enqueue(queue, queued_at, harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value, old_value)
-        Resque.enqueue(ChangeLogs,queued_at, harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value, old_value)
-      end
-    end
-
-    def initialize(args)
-
-      @debug = args.fetch(:debug) { false }
-      @log = args.fetch(:logger) { Logger.new(STDOUT) }
-
-      raise "arguments must contain :crop_number => some_integer" if args[:crop_number].nil?
-
-      resque_class_name = "Distribute"
-
-      # If the class does not exist, the rescue block will create it.
-      # The Class Queue is inherited by the AddQueue, ChangeQueue and DeleteQueue, but
-      # we only want to create one "resque" queue for this instantiation
-      begin
-        Object.const_get(resque_class_name)
-      rescue
-        # Set the queue name to this apol_harvester's id prefixed with a Q_
-        #Object.const_set(resque_class_name, Class.new { @queue =  "Q_#{args[:crop_number]}"})
-        Object.const_set(resque_class_name, Class.new { @queue =  "Farmer"})
-      end
-
-      @resque_queue = Object.const_get(resque_class_name)
-
-    end
-
-    def push(harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value, old_value=Hash.new)
-      check_key primary_key
-
-      ResqueQueue.new(@resque_queue, DateTime.now, harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value.to_json, old_value.to_json)
-
-      message = "RedisQueue: #{@resque_queue}, Now: #{DateTime.now}, Harvester:#{harvester_uuid}, Change:#{crop_change_uuid} crop_number: #{crop_number}, key: #{primary_key}, transaction_type: #{transaction_type})"
-
-      if @log.level == Logger::DEBUG
-        message += ", value: #{value.to_json}, old_value: #{old_value.to_json}"
-        @log.debug message
-      end
-    end
-
-    # Flush any data if needed.
-    #
-    def flush
-    end
-
-    private
-
-    # Raise an exception here if the key must conform to a specific format
-    #
-    def check_key(primary_key)
-      # Example: raise "key must be a string object" unless key.is_a? String
-      primary_key
-    end
-
-  end
-
-  # AddQueue
-  #
-  class AddQueue < Queue
-  end
-
-  # ChangeQueue
-  #
-  class ChangeQueue < Queue
-  end
-
-  # DeleteQueue
-  #
-  class DeleteQueue < Queue
-  end
-
-end