barnyard_harvester 0.0.8 → 0.0.9

data/README.md

@@ -2,6 +2,8 @@
 
 The Harvester gem provides a simple interface where you can iterate your data source and send records to the sync engine.  The default backend storage is Redis.  Since the workers will use Resque which requires Redis, this make sense to use Redis to cache the data.  However, any backend cache can be implemented.
 
+I've now implemented using DynamoDB for the backend and SQS for the messages.
+
 Your data sources are called "crops".  You must assign a unique integer 1..100 to each crop.  This is the integer that is used to create the Redis collection.  By default Redis only allows a maximum of 16 databases, so this must be changed if you go above 16.
 
 WARNING!  Do not use crop number 1-9 in production, they are reserved for system testing.

data/lib/barnyard_harvester/hash_queue.rb

@@ -13,12 +13,12 @@ module BarnyardHarvester
 
     end
 
-    def push(harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value, old_value=Hash.new)
+    def push(harvester_uuid, change_uuid, crop_number, primary_key, transaction_type, value, old_value=Hash.new)
       check_key primary_key
 
       @queue[primary_key] = value.to_json
 
-      @log.debug "HashQueue: Now: #{DateTime.now}, Harvester:#{harvester_uuid}, Change:#{crop_change_uuid} crop_number: #{crop_number}, key: #{primary_key}, transaction_type: #{transaction_type})"
+      @log.debug "HashQueue: Now: #{DateTime.now}, Harvester:#{harvester_uuid}, Change:#{change_uuid} crop_number: #{crop_number}, key: #{primary_key}, transaction_type: #{transaction_type})"
 
     end
 

data/lib/barnyard_harvester/rabbitmq_queue.rb

@@ -9,12 +9,12 @@ module BarnyardHarvester
 
   class Queue
 
-    def enqueue(queue, queued_at, harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value, old_value)
+    def enqueue(queue, queued_at, harvester_uuid, change_uuid, crop_number, primary_key, transaction_type, value, old_value)
 
       payload = Hash.new
       payload[:queued_at] = queued_at
       payload[:harvester_uuid] = harvester_uuid
-      payload[:crop_change_uuid] = crop_change_uuid
+      payload[:change_uuid] = change_uuid
       payload[:crop_number] = crop_number
       payload[:primary_key] = primary_key
       payload[:transaction_type] = transaction_type
@@ -59,12 +59,12 @@ module BarnyardHarvester
 
     end
 
-    def push(harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value, old_value=Hash.new)
+    def push(harvester_uuid, change_uuid, crop_number, primary_key, transaction_type, value, old_value=Hash.new)
       check_key primary_key
 
-      enqueue(QUEUE_FARMER, DateTime.now, harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value.to_json, old_value.to_json)
+      enqueue(QUEUE_FARMER, DateTime.now, harvester_uuid, change_uuid, crop_number, primary_key, transaction_type, value.to_json, old_value.to_json)
 
-      message = "RabbitQueue: #{QUEUE_FARMER}, Now: #{DateTime.now}, Harvester:#{harvester_uuid}, Change:#{crop_change_uuid} crop_number: #{crop_number}, key: #{primary_key}, transaction_type: #{transaction_type})"
+      message = "RabbitQueue: #{QUEUE_FARMER}, Now: #{DateTime.now}, Harvester:#{harvester_uuid}, Change:#{change_uuid} crop_number: #{crop_number}, key: #{primary_key}, transaction_type: #{transaction_type})"
 
       if @log.level == Logger::DEBUG
         message += ", value: #{value.to_json}, old_value: #{old_value.to_json}"

data/lib/barnyard_harvester/resque_queue.rb

@@ -18,9 +18,9 @@ module BarnyardHarvester
   class Queue
 
     class Enqueue
-      def initialize(queue, queued_at, harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value, old_value)
-        Resque.enqueue(queue, queued_at, harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value, old_value)
-        Resque.enqueue(ChangeLogs, queued_at, harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value, old_value)
+      def initialize(queue, queued_at, harvester_uuid, change_uuid, crop_number, primary_key, transaction_type, value, old_value)
+        Resque.enqueue(queue, queued_at, harvester_uuid, change_uuid, crop_number, primary_key, transaction_type, value, old_value)
+        Resque.enqueue(ChangeLogs, queued_at, harvester_uuid, change_uuid, crop_number, primary_key, transaction_type, value, old_value)
       end
     end
 
@@ -54,12 +54,12 @@ module BarnyardHarvester
     end
 
 
-    def push(harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value, old_value=Hash.new)
+    def push(harvester_uuid, change_uuid, crop_number, primary_key, transaction_type, value, old_value=Hash.new)
       check_key primary_key
 
-      Enqueue.new(@resque_queue, DateTime.now, harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value.to_json, old_value.to_json)
+      Enqueue.new(@resque_queue, DateTime.now, harvester_uuid, change_uuid, crop_number, primary_key, transaction_type, value.to_json, old_value.to_json)
 
-      message = "RedisQueue: #{@resque_queue}, Now: #{DateTime.now}, Harvester:#{harvester_uuid}, Change:#{crop_change_uuid} crop_number: #{crop_number}, key: #{primary_key}, transaction_type: #{transaction_type})"
+      message = "RedisQueue: #{@resque_queue}, Now: #{DateTime.now}, Harvester:#{harvester_uuid}, Change:#{change_uuid} crop_number: #{crop_number}, key: #{primary_key}, transaction_type: #{transaction_type})"
 
       if @log.level == Logger::DEBUG
         message += ", value: #{value.to_json}, old_value: #{old_value.to_json}"

data/lib/barnyard_harvester/sqs_queue.rb

@@ -9,12 +9,12 @@ module BarnyardHarvester
 
   class Queue
 
-    def enqueue(queued_at, harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value, old_value)
+    def enqueue(queued_at, harvester_uuid, change_uuid, crop_number, primary_key, transaction_type, value, old_value)
 
       payload = Hash.new
       payload[:queued_at] = queued_at
       payload[:harvester_uuid] = harvester_uuid
-      payload[:crop_change_uuid] = crop_change_uuid
+      payload[:change_uuid] = change_uuid
       payload[:crop_number] = crop_number
       payload[:primary_key] = primary_key
       payload[:transaction_type] = transaction_type
@@ -65,12 +65,12 @@ module BarnyardHarvester
 
     end
 
-    def push(harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value, old_value=Hash.new)
+    def push(harvester_uuid, change_uuid, crop_number, primary_key, transaction_type, value, old_value=Hash.new)
       check_key primary_key
 
-      enqueue(DateTime.now, harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value.to_json, old_value.to_json)
+      enqueue(DateTime.now, harvester_uuid, change_uuid, crop_number, primary_key, transaction_type, value.to_json, old_value.to_json)
 
-      message = "SQS: #{QUEUE_FARMER}, Now: #{DateTime.now}, Harvester:#{harvester_uuid}, Change:#{crop_change_uuid} crop_number: #{crop_number}, key: #{primary_key}, transaction_type: #{transaction_type})"
+      message = "SQS: #{QUEUE_FARMER}, Now: #{DateTime.now}, Harvester:#{harvester_uuid}, Change:#{change_uuid} crop_number: #{crop_number}, key: #{primary_key}, transaction_type: #{transaction_type})"
 
       if @log.level == Logger::DEBUG
         message += ", value: #{value.to_json}, old_value: #{old_value.to_json}"

data/lib/barnyard_harvester/version.rb

@@ -1,3 +1,3 @@
 module BarnyardHarvester
-  VERSION = "0.0.8"
+  VERSION = "0.0.9"
 end

data/lib/barnyard_harvester.rb

@@ -76,8 +76,8 @@ module BarnyardHarvester
 
         # We got delete
         begin
-          crop_change_uuid = @uuid.generate
-          @my_delete_queue.push @harvester_uuid, crop_change_uuid, @crop_number, primary_key, BarnyardHarvester::DELETE, value
+          change_uuid = @uuid.generate
+          @my_delete_queue.push @harvester_uuid, change_uuid, @crop_number, primary_key, BarnyardHarvester::DELETE, value
         rescue Exception => e
           @log.fatal "FATAL error pushing delete #{primary_key} to queue. #{e}"
           exit 1
@@ -99,7 +99,7 @@ module BarnyardHarvester
 
       @source_count += 1
 
-      crop_change_uuid = @uuid.generate
+      change_uuid = @uuid.generate
 
       @key_store[primary_key] = :present # TODO What did this do: if @call_back.nil?
 
@@ -114,7 +114,7 @@ module BarnyardHarvester
         if @my_barn[primary_key] != Crack::JSON.parse(value.to_json)
           #We got change!
           begin
-            @my_change_queue.push(@harvester_uuid, crop_change_uuid, @crop_number, primary_key, BarnyardHarvester::CHANGE, value, @my_barn[primary_key])
+            @my_change_queue.push(@harvester_uuid, change_uuid, @crop_number, primary_key, BarnyardHarvester::CHANGE, value, @my_barn[primary_key])
           rescue Exception => e
             @log.fatal "FATAL error pushing change #{primary_key} to queue. #{e}"
             exit 1
@@ -126,7 +126,7 @@ module BarnyardHarvester
       else
         # We got add!
         #begin
-          @my_add_queue.push(@harvester_uuid, crop_change_uuid, @crop_number, primary_key, BarnyardHarvester::ADD, value)
+          @my_add_queue.push(@harvester_uuid, change_uuid, @crop_number, primary_key, BarnyardHarvester::ADD, value)
         #rescue Exception => e
         #  @log.fatal "FATAL error pushing add #{primary_key} to queue. #{e}"
         #  exit 1

data/spec/dynamodb_sqs_spec.rb

@@ -4,7 +4,7 @@ require "logger"
 require "json"
 require "aws-sdk"
 
-CROP_NUMBER = 1
+CROP_NUMBER = 10
 
 DEFAULT_DYNAMODB_SETTINGS = {
     :dynamo_db_endpoint => "dynamodb.us-west-1.amazonaws.com",
@@ -57,6 +57,8 @@ describe BarnyardHarvester do
       table = db.tables[table_name]
       table.hash_key = [:id, :string]
     end
+
+    # TODO AWS::SQS::Errors::QueueDeletedRecently wait 60 seconds and retry
     table
   end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: barnyard_harvester
 version: !ruby/object:Gem::Version
-  version: 0.0.8
+  version: 0.0.9
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-11-27 00:00:00.000000000 Z
+date: 2012-11-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70097767091020 !ruby/object:Gem::Requirement
+  requirement: &70360766873440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70097767091020
+  version_requirements: *70360766873440
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70097767086260 !ruby/object:Gem::Requirement
+  requirement: &70360766873020 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70097767086260
+  version_requirements: *70360766873020
 - !ruby/object:Gem::Dependency
   name: crack
-  requirement: &70097767074760 !ruby/object:Gem::Requirement
+  requirement: &70360766872600 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70097767074760
+  version_requirements: *70360766872600
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70097767067660 !ruby/object:Gem::Requirement
+  requirement: &70360766872180 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70097767067660
+  version_requirements: *70360766872180
 - !ruby/object:Gem::Dependency
   name: uuid
-  requirement: &70097767061320 !ruby/object:Gem::Requirement
+  requirement: &70360766871760 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,10 +65,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70097767061320
+  version_requirements: *70360766871760
 - !ruby/object:Gem::Dependency
   name: bson_ext
-  requirement: &70097767057920 !ruby/object:Gem::Requirement
+  requirement: &70360766871260 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - =
@@ -76,10 +76,10 @@ dependencies:
         version: 1.6.0
   type: :runtime
   prerelease: false
-  version_requirements: *70097767057920
+  version_requirements: *70360766871260
 - !ruby/object:Gem::Dependency
   name: mongo
-  requirement: &70097767055100 !ruby/object:Gem::Requirement
+  requirement: &70360766870760 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - =
@@ -87,10 +87,10 @@ dependencies:
         version: 1.6.0
   type: :runtime
   prerelease: false
-  version_requirements: *70097767055100
+  version_requirements: *70360766870760
 - !ruby/object:Gem::Dependency
   name: amqp
-  requirement: &70097767053440 !ruby/object:Gem::Requirement
+  requirement: &70360766870380 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -98,10 +98,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70097767053440
+  version_requirements: *70360766870380
 - !ruby/object:Gem::Dependency
   name: aws-sdk
-  requirement: &70097767049260 !ruby/object:Gem::Requirement
+  requirement: &70360766869920 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -109,7 +109,7 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70097767049260
+  version_requirements: *70360766869920
 description: Performs harvests on data sources and detects adds, changes and deletes.
 email:
 - supercoder@gmail.com
@@ -172,7 +172,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: barnyard_harvester
-rubygems_version: 1.8.15
+rubygems_version: 1.8.11
 signing_key: 
 specification_version: 3
 summary: Please check the README.md for more information.