barnyard_harvester 0.0.10 → 0.0.12

data/barnyard_harvester.gemspec

@@ -27,6 +27,7 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "bson_ext", "= 1.6.0"
   gem.add_runtime_dependency "mongo", "= 1.6.0"
   gem.add_runtime_dependency "amqp"
+  gem.add_runtime_dependency "bunny"
   gem.add_runtime_dependency "aws-sdk"
 
 end

data/lib/barnyard_harvester/generic_queue.rb

@@ -0,0 +1,72 @@
+module BarnyardHarvester
+
+  class GenericQueue
+
+    def initialize(args)
+      @queueing = args.fetch(:queueing) { raise "You must provide :queueing" }
+
+      case @queueing
+        when :sqs
+          require "aws-sdk"
+          @sqs_settings = args.fetch(:sqs_settings) { raise "You must provide :sqs_settings" }
+          @sqs = AWS::SQS.new(@sqs_settings)
+        when :rabbitmq
+          require "bunny"
+          @rabbitmq_settings = args.fetch(:rabbitmq_settings) { raise "You must provide :rabbitmq_settings" }
+          @rabbitmq_settings[:logging] = true if @debug
+          @bunny = Bunny.new(@rabbitmq_settings)
+          @bunny.start
+        when :hash
+          @queues = Hash.new
+        else
+          raise "Unknown queueing method. #{@queuing}"
+      end
+
+    end
+
+    def push(queue_name, message)
+      case @queueing
+        when :sqs
+          queue = @sqs.queues.create(queue_name)
+          queue.send_message(message)
+        when :rabbitmq
+          @bunny.queue(queue_name).publish(message)
+        when :hash
+          @queues[queue_name] = Array.new unless @queues.has_key?(queue_name)
+          @queues[queue_name] << message
+          File.open("#{queue_name}.yml", "w") { |file| file.puts(@queues[queue_name].to_yaml) }
+      end
+    end
+
+    def pop(queue_name)
+      case @queueing
+        when :sqs
+          msg = @sqs.queues.create(queue_name).receive_message
+          unless msg.nil?
+            msg.delete
+            msg.body
+          else
+            nil
+          end
+
+        when :rabbitmq
+          msg = @bunny.queue(queue_name).pop[:payload]
+          if msg == :queue_empty
+            return nil
+          else
+            msg
+          end
+        when :hash
+          msg = @queue.pop
+          File.open("#{queue_name}.yml", "w") { |file| file.puts(@queues[queue_name].to_yaml) }
+          msg
+      end
+    end
+
+    def empty(queue_name)
+      while pop(queue_name)
+      end
+    end
+  end
+
+end

data/lib/barnyard_harvester/mongodb.rb

@@ -42,15 +42,15 @@ module BarnyardHarvester
       @collection
     end
 
-    def log_run(harvester_uuid, crop_number, began_at, ended_at, source_count, change_count, add_count, delete_count)
-
-      begin
-        Resque.enqueue(HarvesterLogs, Time.now, harvester_uuid, crop_number, began_at, ended_at, source_count, change_count, add_count, delete_count)
-      rescue Exception => e
-        logger.fatal "#{self.class} Fail in Resque.enqueue of HarvesterLogs. #{e.backtrace}"
-      end
-
-    end
+    #def log_run(harvester_uuid, crop_number, began_at, ended_at, source_count, change_count, add_count, delete_count)
+    #
+    #  begin
+    #    Resque.enqueue(HarvesterLogs, Time.now, harvester_uuid, crop_number, began_at, ended_at, source_count, change_count, add_count, delete_count)
+    #  rescue Exception => e
+    #    @log.fatal "#{self.class} Fail in Resque.enqueue of HarvesterLogs. #{e.backtrace}"
+    #  end
+    #
+    #end
 
     def delete(primary_key)
       check_key primary_key

data/lib/barnyard_harvester/{rabbitmq_queue.rb → queue.rb}

@@ -1,6 +1,7 @@
-module BarnyardHarvester
 
-  require "bunny"
+require "barnyard_harvester/generic_queue"
+
+module BarnyardHarvester
 
   QUEUE_FARMER = "barnyard-farmer"
   QUEUE_HARVESTER = "barnyard-harvests"
@@ -9,10 +10,10 @@ module BarnyardHarvester
 
   class Queue
 
-    def enqueue(queue, queued_at, harvester_uuid, change_uuid, crop_number, primary_key, transaction_type, value, old_value)
+    def enqueue(queue, harvester_uuid, change_uuid, crop_number, primary_key, transaction_type, value, old_value)
 
       payload = Hash.new
-      payload[:queued_at] = queued_at
+      payload[:queued_at] = Time.now
       payload[:harvester_uuid] = harvester_uuid
       payload[:change_uuid] = change_uuid
       payload[:crop_number] = crop_number
@@ -21,10 +22,10 @@ module BarnyardHarvester
       payload[:value] = value
       payload[:old_value] = old_value
 
-      json_payload = payload
+      json_payload = payload.to_json
 
-      @direct_exchange.publish(json_payload, key: queue)
-      @direct_exchange.publish(json_payload, key: QUEUE_CHANGE)
+      @q.push(queue,json_payload)
+      @q.push(QUEUE_CHANGE,json_payload)
 
     end
 
@@ -41,7 +42,7 @@ module BarnyardHarvester
       payload[:add_count] = add_count
       payload[:delete_count] = delete_count
 
-      @direct_exchange.publish(payload.to_json, key: QUEUE_HARVESTER)
+      @q.push(QUEUE_HARVESTER,payload.to_json)
 
     end
 
@@ -50,19 +51,15 @@ module BarnyardHarvester
       @debug = args.fetch(:debug) { false }
       @log = args.fetch(:logger) { Logger.new(STDOUT) }
       @crop_number = args.fetch(:crop_number) { raise "You must provide :crop_number" }
-      @rabbitmq_settings = args.fetch(:rabbitmq_settings) { raise "You must provide :rabbitmq_settings" }
-
-      @bunny = Bunny.new(@rabbitmq_settings)
-      @bunny.start
 
-      @direct_exchange = @bunny.exchange('');
+      @q = BarnyardHarvester::GenericQueue.new(args)
 
     end
 
     def push(harvester_uuid, change_uuid, crop_number, primary_key, transaction_type, value, old_value=Hash.new)
       check_key primary_key
 
-      enqueue(QUEUE_FARMER, DateTime.now, harvester_uuid, change_uuid, crop_number, primary_key, transaction_type, value.to_json, old_value.to_json)
+      enqueue(QUEUE_FARMER, harvester_uuid, change_uuid, crop_number, primary_key, transaction_type, value.to_json, old_value.to_json)
 
       message = "RabbitQueue: #{QUEUE_FARMER}, Now: #{DateTime.now}, Harvester:#{harvester_uuid}, Change:#{change_uuid} crop_number: #{crop_number}, key: #{primary_key}, transaction_type: #{transaction_type})"
 

data/lib/barnyard_harvester/version.rb

@@ -1,3 +1,3 @@
 module BarnyardHarvester
-  VERSION = "0.0.10"
+  VERSION = "0.0.12"
 end

data/lib/barnyard_harvester.rb

@@ -2,6 +2,7 @@ require "uuid"
 require "logger"
 
 require "barnyard_harvester/version"
+require "barnyard_harvester/queue"
 
 module BarnyardHarvester
 
@@ -20,21 +21,24 @@ module BarnyardHarvester
 
     def initialize(args)
 
+      @queueing = args.fetch(:queueing) { raise "You must provide :queueing" }
       @crop_number = args.fetch(:crop_number) { raise "You must provide :crop_number" }
       @redis_settings = args.fetch(:redis_settings) { DEFAULT_REDIS_SETTINGS }
       @debug = args.fetch(:debug) { false }
       @log = args.fetch(:logger) { Logger.new(STDOUT) }
 
-      @queueing = args[:queueing]
+      @q = BarnyardHarvester::Queue.new(args)
 
-      case @queueing
-        when :rabbitmq
-          @rabbitmq_settings = args.fetch(:rabbitmq_settings) { raise "You must provide :rabbitmq_settings" }
-        when :sqs
-          @sqs_settings = args.fetch(:sqs_settings) { raise "You must provide :sqs_settings" }
-        else
-          @queueing = :resque
-      end
+      #@queueing = args[:queueing]
+      #
+      #case @queueing
+      #  when :rabbitmq
+      #    @rabbitmq_settings = args.fetch(:rabbitmq_settings) { raise "You must provide :rabbitmq_settings" }
+      #  when :sqs
+      #    @sqs_settings = args.fetch(:sqs_settings) { raise "You must provide :sqs_settings" }
+      #  else
+      #    @queueing = :resque
+      #end
 
       @backend = args.fetch(:backend) { :redis }
 
@@ -43,7 +47,7 @@ module BarnyardHarvester
       end
 
       require "barnyard_harvester/#{@backend.to_s}_helper" if File.exist? "barnyard_harvester/#{@backend.to_s}_helper"
-      require "barnyard_harvester/#{@queueing.to_s}_queue"
+#      require "barnyard_harvester/#{@queueing.to_s}_queue"
       require "barnyard_harvester/#{@backend.to_s}"
 
 #      YAML::ENGINE.yamler = 'syck'

data/spec/hash_spec.rb

@@ -22,7 +22,7 @@ describe BarnyardHarvester do
     my_logger = Logger.new(STDOUT)
     my_logger.level = Logger::INFO
 
-    h = BarnyardHarvester::Sync.new(:queueing => :resque,
+    h = BarnyardHarvester::Sync.new(:queueing => :hash,
                                     :backend => backend,
                                     :debug => false,
                                     :crop_number => 1,

data/spec/loader_spec.rb

@@ -1,5 +1,10 @@
 require "barnyard_harvester"
 
+RABBITMQ_SETTINGS = {
+    :host => "localhost"
+    #    :port => 6163
+}
+
 describe BarnyardHarvester do
 
   it "no parameters should raise error" do
@@ -7,31 +12,31 @@ describe BarnyardHarvester do
   end
 
   it "passing only :crop_number => 1 should return BarnyardHarvester::Sync object" do
-    BarnyardHarvester::Sync.new(crop_number: 1).class.should eq(BarnyardHarvester::Sync)
+    BarnyardHarvester::Sync.new(rabbitmq_settings: RABBITMQ_SETTINGS, queueing: :rabbitmq, crop_number: 1).class.should eq(BarnyardHarvester::Sync)
   end
 
   it "default backend should be :redis" do
-    BarnyardHarvester::Sync.new(crop_number: 1).backend.should eq(:redis)
+    BarnyardHarvester::Sync.new(rabbitmq_settings: RABBITMQ_SETTINGS, queueing: :rabbitmq, crop_number: 1).backend.should eq(:redis)
   end
 
   it "passing backend :hash should be :hash" do
-    BarnyardHarvester::Sync.new(crop_number: 1, backend: :hash).backend.should eq(:hash)
+    BarnyardHarvester::Sync.new(rabbitmq_settings: RABBITMQ_SETTINGS, queueing: :rabbitmq, crop_number: 1, backend: :hash).backend.should eq(:hash)
   end
 
   it "passing bogus backend should raise an error" do
-    lambda{BarnyardHarvester::Sync.new(crop_number: 1, backend: :foobar)}.should raise_error
+    lambda{BarnyardHarvester::Sync.new(rabbitmq_settings: RABBITMQ_SETTINGS, queueing: :rabbitmq, crop_number: 1, backend: :foobar)}.should raise_error
   end
 
   it "crop_number should be 1001" do
-    BarnyardHarvester::Sync.new(crop_number: 1001).crop_number.should eq(1001)
+    BarnyardHarvester::Sync.new(rabbitmq_settings: RABBITMQ_SETTINGS, queueing: :rabbitmq, crop_number: 1001).crop_number.should eq(1001)
   end
 
   it "all counters should be zero" do
-    BarnyardHarvester::Sync.new(crop_number: 1).change_count.should eq(0)
-    BarnyardHarvester::Sync.new(crop_number: 1).add_count.should eq(0)
-    BarnyardHarvester::Sync.new(crop_number: 1).delete_count.should eq(0)
-    BarnyardHarvester::Sync.new(crop_number: 1).source_count.should eq(0)
-    BarnyardHarvester::Sync.new(crop_number: 1).cache_count.should eq(0)
+    BarnyardHarvester::Sync.new(rabbitmq_settings: RABBITMQ_SETTINGS, queueing: :rabbitmq, crop_number: 1).change_count.should eq(0)
+    BarnyardHarvester::Sync.new(rabbitmq_settings: RABBITMQ_SETTINGS, queueing: :rabbitmq, crop_number: 1).add_count.should eq(0)
+    BarnyardHarvester::Sync.new(rabbitmq_settings: RABBITMQ_SETTINGS, queueing: :rabbitmq, crop_number: 1).delete_count.should eq(0)
+    BarnyardHarvester::Sync.new(rabbitmq_settings: RABBITMQ_SETTINGS, queueing: :rabbitmq, crop_number: 1).source_count.should eq(0)
+    BarnyardHarvester::Sync.new(rabbitmq_settings: RABBITMQ_SETTINGS, queueing: :rabbitmq, crop_number: 1).cache_count.should eq(0)
   end
 
 end

data/spec/{mongo_spec.rb → mongo_rabbitmq_spec.rb}

@@ -7,12 +7,6 @@ require "json"
 
 CROP_NUMBER = 1
 
-REDIS_SETTINGS = {
-    :host => "localhost",
-    :port => 6379,
-    :db => CROP_NUMBER
-}
-
 MONGODB_SETTINGS = {
     :host => "localhost",
     :collection => "test_collection",
@@ -21,14 +15,19 @@ MONGODB_SETTINGS = {
 }
 
 MONGODB_REPLICA_SET_SETTINGS = {
-    :host => ["ip-172-19-31-44.c.qaapollogrp.edu:27017", "ip-172-19-30-49.c.qaapollogrp.edu:27017", "ip-172-19-31-202.c.qaapollogrp.edu:27017"],
+    :host => ["mongo1:27017", "mongo2:27017", "mongo3:27017"],
     :collection => "test_collection",
     :db => "test_database",
-    :user => "honeybadger",
-    :password => "0joQuk35vJM05Hj",
+    :user => "username",
+    :password => "password",
     :collection => "test_collection"
 }
 
+RABBITMQ_SETTINGS = {
+    :host => "localhost"
+#    :port => 6163
+}
+
 $mongo_settings = MONGODB_SETTINGS
 
 describe BarnyardHarvester do
@@ -41,11 +40,11 @@ describe BarnyardHarvester do
     my_logger.level = Logger::INFO
 
     h = BarnyardHarvester::Sync.new(:backend => backend,
-                                    :queueing => :resque,
+                                    :queueing => :rabbitmq,
                                     :debug => false,
                                     :mongodb_settings => $mongo_settings,
                                     :crop_number => CROP_NUMBER,
-                                    :redis_settings => REDIS_SETTINGS,
+                                    :rabbitmq_settings => RABBITMQ_SETTINGS,
                                     :logger => my_logger)
 
     h.run do
@@ -125,8 +124,6 @@ describe BarnyardHarvester do
     h.source_count.should eq(data.count)
     h.cache_count.should eq(data.count)
 
-    h.my_barn.log_run("#{file}-#{Random.rand(100)}", @crop_number, Time.now, Time.now, h.source_count, h.change_count, h.add_count, h.delete_count)
-
   end
 
   it "test change one record" do
@@ -143,8 +140,6 @@ describe BarnyardHarvester do
     h.source_count.should eq(data.count)
     h.cache_count.should eq(data.count)
 
-    h.my_barn.log_run("#{file}-#{Random.rand(100)}", @crop_number, Time.now, Time.now, h.source_count, h.change_count, h.add_count, h.delete_count)
-
   end
 
   it "test delete one record" do
@@ -161,8 +156,6 @@ describe BarnyardHarvester do
     h.source_count.should eq(data.count)
     h.cache_count.should eq(data.count + 1)
 
-    h.my_barn.log_run("#{file}-#{Random.rand(100)}", @crop_number, Time.now, Time.now, h.source_count, h.change_count, h.add_count, h.delete_count)
-
   end
 
   it "test delete all records and add one" do
@@ -181,8 +174,6 @@ describe BarnyardHarvester do
     h.source_count.should eq(1)
     h.cache_count.should eq(init_data.count + 1)
 
-    h.my_barn.log_run("#{file}-#{Random.rand(100)}", @crop_number, Time.now, Time.now, h.source_count, h.change_count, h.add_count, h.delete_count)
-
   end
 
 

data/spec/queue_rabbitmq_spec.rb

@@ -0,0 +1,37 @@
+require "barnyard_harvester"
+
+RABBITMQ_SETTINGS = {
+    :host => "localhost"
+    #    :port => 6163
+}
+
+QUEUE_NAME = "test001"
+
+describe "Test RabbitMQ" do
+
+  before(:each) do
+
+    @q = BarnyardHarvester::GenericQueue.new(queueing: :rabbitmq, rabbitmq_settings: RABBITMQ_SETTINGS)
+
+  end
+
+  it "Push 10 items on the queue" do
+
+    @q.empty(QUEUE_NAME)
+
+    10.times do |i|
+      @q.push(QUEUE_NAME, "My Message #{i}")
+    end
+
+  end
+
+  it "Pop 10 items" do
+
+    10.times do |i|
+      msg = @q.pop(QUEUE_NAME)
+      puts msg
+    end
+
+  end
+
+end

data/spec/queue_sqs_spec.rb

@@ -0,0 +1,39 @@
+require "barnyard_harvester"
+
+SQS_SETTINGS = {
+    :sqs_endpoint => "sqs.us-west-1.amazonaws.com",
+    :access_key_id => ENV["AWS_ACCESS_KEY_ID"],
+    :secret_access_key => ENV["AWS_SECRET_ACCESS_KEY"]
+}
+
+QUEUE_NAME = "test001"
+
+describe "Test SQS" do
+
+  before(:each) do
+
+    @q = BarnyardHarvester::GenericQueue.new(queueing: :sqs, sqs_settings: SQS_SETTINGS)
+
+  end
+
+  it "Push 10 items on the queue" do
+
+    @q.empty(QUEUE_NAME)
+
+    10.times do |i|
+      @q.push(QUEUE_NAME, "My Message #{i}")
+    end
+
+  end
+
+  it "Pop 10 items" do
+
+  10.times do |i|
+      msg = @q.pop(QUEUE_NAME)
+      puts msg
+    end
+
+  end
+
+
+end

data/spec/{redis_spec.rb → redis_hash_spec.rb}

@@ -21,7 +21,7 @@ describe BarnyardHarvester do
     my_logger = Logger.new(STDOUT)
     my_logger.level = Logger::INFO
 
-    h = BarnyardHarvester::Sync.new(:backend => backend, :debug => false, :crop_number => CROP_NUMBER, :redis_settings => REDIS_SETTINGS, :logger => my_logger)
+    h = BarnyardHarvester::Sync.new(:queueing => :hash, :backend => backend, :debug => false, :crop_number => CROP_NUMBER, :redis_settings => REDIS_SETTINGS, :logger => my_logger)
 
     h.run do
       data.each do |primary_key, value|

data/spec/redis_rabbitmq_spec.rb

@@ -93,8 +93,6 @@ describe BarnyardHarvester do
     h.source_count.should eq(data.count)
     h.cache_count.should eq(data.count)
 
-#    h.log_run("#{file}-#{Random.rand(100)}", @crop_number, Time.now, Time.now, h.source_count, h.change_count, h.add_count, h.delete_count)
-
   end
 
   it "test change one record" do
@@ -111,8 +109,6 @@ describe BarnyardHarvester do
     h.source_count.should eq(data.count)
     h.cache_count.should eq(data.count)
 
-#    h.my_barn.log_run("#{file}-#{Random.rand(100)}", @crop_number, Time.now, Time.now, h.source_count, h.change_count, h.add_count, h.delete_count)
-
   end
 
   it "test delete one record" do
@@ -150,13 +146,10 @@ describe BarnyardHarvester do
     h.source_count.should eq(1)
     h.cache_count.should eq(init_data.count + 1)
 
-#    h.my_barn.log_run("#{file}-#{Random.rand(100)}", @crop_number, Time.now, Time.now, h.source_count, h.change_count, h.add_count, h.delete_count)
-
   end
 
 
   after(:each) do
   end
 
-
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: barnyard_harvester
 version: !ruby/object:Gem::Version
-  version: 0.0.10
+  version: 0.0.12
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-11-30 00:00:00.000000000 Z
+date: 2012-12-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70321910695840 !ruby/object:Gem::Requirement
+  requirement: &70330836174420 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70321910695840
+  version_requirements: *70330836174420
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70321910691160 !ruby/object:Gem::Requirement
+  requirement: &70330836173200 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70321910691160
+  version_requirements: *70330836173200
 - !ruby/object:Gem::Dependency
   name: crack
-  requirement: &70321910677340 !ruby/object:Gem::Requirement
+  requirement: &70330836169660 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70321910677340
+  version_requirements: *70330836169660
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70321910671040 !ruby/object:Gem::Requirement
+  requirement: &70330836167600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70321910671040
+  version_requirements: *70330836167600
 - !ruby/object:Gem::Dependency
   name: uuid
-  requirement: &70321910666000 !ruby/object:Gem::Requirement
+  requirement: &70330836182240 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70321910666000
+  version_requirements: *70330836182240
 - !ruby/object:Gem::Dependency
   name: bson_ext
-  requirement: &70321910662080 !ruby/object:Gem::Requirement
+  requirement: &70330836179360 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - =
@@ -76,10 +76,10 @@ dependencies:
         version: 1.6.0
   type: :runtime
   prerelease: false
-  version_requirements: *70321910662080
+  version_requirements: *70330836179360
 - !ruby/object:Gem::Dependency
   name: mongo
-  requirement: &70321910659980 !ruby/object:Gem::Requirement
+  requirement: &70330836189880 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - =
@@ -87,10 +87,10 @@ dependencies:
         version: 1.6.0
   type: :runtime
   prerelease: false
-  version_requirements: *70321910659980
+  version_requirements: *70330836189880
 - !ruby/object:Gem::Dependency
   name: amqp
-  requirement: &70321910658560 !ruby/object:Gem::Requirement
+  requirement: &70330836188800 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,21 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70321910658560
+  version_requirements: *70330836188800
+- !ruby/object:Gem::Dependency
+  name: bunny
+  requirement: &70330836187660 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70330836187660
 - !ruby/object:Gem::Dependency
   name: aws-sdk
-  requirement: &70321910654380 !ruby/object:Gem::Requirement
+  requirement: &70330836186540 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,7 +120,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70321910654380
+  version_requirements: *70330836186540
 description: Performs harvests on data sources and detects adds, changes and deletes.
 email:
 - supercoder@gmail.com
@@ -128,14 +139,13 @@ files:
 - lib/.DS_Store
 - lib/barnyard_harvester.rb
 - lib/barnyard_harvester/dynamodb.rb
+- lib/barnyard_harvester/generic_queue.rb
 - lib/barnyard_harvester/hash.rb
 - lib/barnyard_harvester/hash_queue.rb
 - lib/barnyard_harvester/mongodb.rb
 - lib/barnyard_harvester/mongodb_helper.rb
-- lib/barnyard_harvester/rabbitmq_queue.rb
+- lib/barnyard_harvester/queue.rb
 - lib/barnyard_harvester/redis.rb
-- lib/barnyard_harvester/resque_queue.rb
-- lib/barnyard_harvester/sqs_queue.rb
 - lib/barnyard_harvester/version.rb
 - lib/test.yml
 - spec/dynamodb_sqs_spec.rb
@@ -147,9 +157,11 @@ files:
 - spec/hash_spec.rb
 - spec/loader_spec.rb
 - spec/mongo_helper_spec.rb
-- spec/mongo_spec.rb
+- spec/mongo_rabbitmq_spec.rb
+- spec/queue_rabbitmq_spec.rb
+- spec/queue_sqs_spec.rb
+- spec/redis_hash_spec.rb
 - spec/redis_rabbitmq_spec.rb
-- spec/redis_spec.rb
 - spec/redis_sqs_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/jongillies/barnyard/tree/master/barnyard_harvester
@@ -186,8 +198,10 @@ test_files:
 - spec/hash_spec.rb
 - spec/loader_spec.rb
 - spec/mongo_helper_spec.rb
-- spec/mongo_spec.rb
+- spec/mongo_rabbitmq_spec.rb
+- spec/queue_rabbitmq_spec.rb
+- spec/queue_sqs_spec.rb
+- spec/redis_hash_spec.rb
 - spec/redis_rabbitmq_spec.rb
-- spec/redis_spec.rb
 - spec/redis_sqs_spec.rb
 - spec/spec_helper.rb

data/lib/barnyard_harvester/resque_queue.rb

@@ -1,101 +0,0 @@
-module BarnyardHarvester
-
-  class ChangeLogs
-    @queue = :logs_change
-  end
-
-  class HarvesterLogs
-    @queue = :logs_harvester
-  end
-
-  class DeliveryLogs
-    @queue = :logs_delivery
-  end
-
-  class TransactionLogs
-    @queue = :logs_transaction
-  end
-  class Queue
-
-    class Enqueue
-      def initialize(queue, queued_at, harvester_uuid, change_uuid, crop_number, primary_key, transaction_type, value, old_value)
-        Resque.enqueue(queue, queued_at, harvester_uuid, change_uuid, crop_number, primary_key, transaction_type, value, old_value)
-        Resque.enqueue(ChangeLogs, queued_at, harvester_uuid, change_uuid, crop_number, primary_key, transaction_type, value, old_value)
-      end
-    end
-
-    def log_run(harvester_uuid, crop_number, began_at, ended_at, source_count, change_count, add_count, delete_count)
-
-      Resque.enqueue(HarvesterLogs, harvester_uuid, crop_number, began_at, ended_at, source_count, change_count, add_count, delete_count)
-
-    end
-
-    def initialize(args)
-
-      @debug = args.fetch(:debug) { false }
-      @log = args.fetch(:logger) { Logger.new(STDOUT) }
-      @crop_number = args.fetch(:crop_number) { raise "You must provide :crop_number" }
-
-      resque_class_name = "Distribute"
-
-      # If the class does not exist, the rescue block will create it.
-      # The Class Queue is inherited by the AddQueue, ChangeQueue and DeleteQueue, but
-      # we only want to create one "resque" queue for this instantiation
-      begin
-        Object.const_get(resque_class_name)
-      rescue
-        # Set the queue name to this apol_harvester's id prefixed with a Q_
-        #Object.const_set(resque_class_name, Class.new { @queue =  "Q_#{args[:crop_number]}"})
-        Object.const_set(resque_class_name, Class.new { @queue = "Farmer" })
-      end
-
-      @resque_queue = Object.const_get(resque_class_name)
-
-    end
-
-
-    def push(harvester_uuid, change_uuid, crop_number, primary_key, transaction_type, value, old_value=Hash.new)
-      check_key primary_key
-
-      Enqueue.new(@resque_queue, DateTime.now, harvester_uuid, change_uuid, crop_number, primary_key, transaction_type, value.to_json, old_value.to_json)
-
-      message = "RedisQueue: #{@resque_queue}, Now: #{DateTime.now}, Harvester:#{harvester_uuid}, Change:#{change_uuid} crop_number: #{crop_number}, key: #{primary_key}, transaction_type: #{transaction_type})"
-
-      if @log.level == Logger::DEBUG
-        message += ", value: #{value.to_json}, old_value: #{old_value.to_json}"
-        @log.debug message
-      end
-    end
-
-    # Flush any data if needed.
-    #
-    def flush
-    end
-
-    private
-
-    # Raise an exception here if the key must conform to a specific format
-    #
-    def check_key(primary_key)
-      # Example: raise "key must be a string object" unless key.is_a? String
-      primary_key
-    end
-
-  end
-
-  # AddQueue
-  #
-  class AddQueue < Queue
-  end
-
-  # ChangeQueue
-  #
-  class ChangeQueue < Queue
-  end
-
-  # DeleteQueue
-  #
-  class DeleteQueue < Queue
-  end
-
-end

data/lib/barnyard_harvester/sqs_queue.rb

@@ -1,112 +0,0 @@
-module BarnyardHarvester
-
-  require "aws-sdk"
-
-  QUEUE_FARMER = "barnyard-farmer"
-  QUEUE_HARVESTER = "barnyard-harvests"
-  QUEUE_TRANSACTION = "barnyard-transactions"
-  QUEUE_CHANGE = "barnyard-changes"
-
-  class Queue
-
-    def enqueue(queued_at, harvester_uuid, change_uuid, crop_number, primary_key, transaction_type, value, old_value)
-
-      payload = Hash.new
-      payload[:queued_at] = queued_at
-      payload[:harvester_uuid] = harvester_uuid
-      payload[:change_uuid] = change_uuid
-      payload[:crop_number] = crop_number
-      payload[:primary_key] = primary_key
-      payload[:transaction_type] = transaction_type
-      payload[:value] = value
-      payload[:old_value] = old_value
-
-      json_payload = payload.to_json
-
-      @log.debug "Starting send_message to farmer..."
-      @farmer_queue.send_message(json_payload)
-      @log.debug "Done send_message to farmer..."
-
-      @log.debug "Starting send_message to changes..."
-      @change_queue.send_message(json_payload)
-      @log.debug "Done send_message to changes..."
-
-    end
-
-    def log_run(harvester_uuid, crop_number, began_at, ended_at, source_count, change_count, add_count, delete_count)
-
-      payload = Hash.new
-      payload[:time] = Time.now
-      payload[:harvester_uuid] = harvester_uuid
-      payload[:crop_number] = crop_number
-      payload[:began_at] = began_at
-      payload[:ended_at] = ended_at
-      payload[:source_count] = source_count
-      payload[:change_count] = change_count
-      payload[:add_count] = add_count
-      payload[:delete_count] = delete_count
-
-      @harvester_queue.send_message(payload.to_json)
-
-    end
-
-    def initialize(args)
-
-      @debug = args.fetch(:debug) { false }
-      @log = args.fetch(:logger) { Logger.new(STDOUT) }
-      @crop_number = args.fetch(:crop_number) { raise "You must provide :crop_number" }
-      @sqs_settings = args.fetch(:sqs_settings) { raise "You must provide :sqs_settings" }
-
-      @sqs = AWS::SQS.new(@sqs_settings)
-
-      @farmer_queue = @sqs.queues.create(QUEUE_FARMER)
-      @harvester_queue = @sqs.queues.create(QUEUE_HARVESTER)
-      @change_queue = @sqs.queues.create(QUEUE_CHANGE)
-
-    end
-
-    def push(harvester_uuid, change_uuid, crop_number, primary_key, transaction_type, value, old_value=Hash.new)
-      check_key primary_key
-
-      enqueue(DateTime.now, harvester_uuid, change_uuid, crop_number, primary_key, transaction_type, value.to_json, old_value.to_json)
-
-      message = "SQS: #{QUEUE_FARMER}, Now: #{DateTime.now}, Harvester:#{harvester_uuid}, Change:#{change_uuid} crop_number: #{crop_number}, key: #{primary_key}, transaction_type: #{transaction_type})"
-
-      if @log.level == Logger::DEBUG
-        message += ", value: #{value.to_json}, old_value: #{old_value.to_json}"
-        @log.debug message
-      end
-    end
-
-    # Flush any data if needed.
-    #
-    def flush
-    end
-
-    private
-
-    # Raise an exception here if the key must conform to a specific format
-    #
-    def check_key(primary_key)
-      # Example: raise "key must be a string object" unless key.is_a? String
-      primary_key
-    end
-
-  end
-
-  # AddQueue
-  #
-  class AddQueue < Queue
-  end
-
-  # ChangeQueue
-  #
-  class ChangeQueue < Queue
-  end
-
-  # DeleteQueue
-  #
-  class DeleteQueue < Queue
-  end
-
-end