barnyard_harvester 0.0.1

data/.gitignore

@@ -0,0 +1,6 @@
+*.gem
+*.idea
+.bundle
+Gemfile.lock
+pkg/*
+1-BarnyardHarvester::*

data/.rvmrc

@@ -0,0 +1 @@
+rvm use ruby-1.9.3-p125@barnyard

data/Gemfile

@@ -0,0 +1,4 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in harvester.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,23 @@
+Copyright (c) 2012 Jon Gillies
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+

data/README.md

@@ -0,0 +1,94 @@
+# BarnyardHarvester
+
+The Harvester gem provides a simple interface where you can iterate your data source and send records to the sync engine.  The default backend storage is Redis.  Since the workers will use Resque which requires Redis, this make sense to use Redis to cache the data.  However, any backend cache can be implemented.
+
+Your data sources are called "crops".  You must assign a unique integer 1..100 to each crop.  This is the integer that is used to create the Redis collection.  By default Redis only allows a maximum of 16 databases, so this must be changed if you go above 16.
+
+WARNING!  Do not use crop number 1-9 in production, they are reserved for system testing.
+
+The sync engine keeps a copy of your data source in a Redis databased indexed on the data source's primary key.  When you sync your data, the engine uses this "cached" copy of the data to determine adds, deletes and changes.  None of the data is inspected and the sync engine does not care what is in the data as long as it can be marshaled into a JSON string.  The sync engine automatically does the conversion to and from JSON, so all you pass in is an object that responds to .to_json.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'barnyard_harvester'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install barnyard_harvester
+
+## Getting Started
+
+This example assumes that you have Redis running on your local box localhost:6379.  If you do not have Redis installed you can add :backend => :hash to the constructor.
+
+    require "barnyard_harvester"
+
+    # Get a connection to your data source
+    # (For this example we will use a YAML file)
+
+    @data = YAML::load_file("test.yml")
+
+    h = BarnyardHarvester::Sync.new(:debug => false, :crop_number => 1,
+                                   #:backend => :hash
+    )
+
+    h.run do
+
+      # Iterate your data here and call #process with the primary key and the value
+      @data.each do |primary_key, value|
+        my_log.info "PK: #{primary_key}"
+        h.process primary_key, value       # <<-- Send your data here!
+      end
+
+    end
+
+    puts h.stats
+
+Sample output:
+
+    (0) adds, (0) deletes, (0) changes, (5) source records, (5) cache records
+
+## Testing
+
+RSPEC is used to test this gem.  Note that you should run each spec test individually.  For some reason they confilict if run together.  You will need Redis installed on the localhost:6379 or adjust the test accordingly.
+
+This tests the initilization and parameters:
+
+    rspec -c spec/loader_spec.rb
+    .......
+
+    Finished in 0.24556 seconds
+    7 examples, 0 failures
+
+This tests the Redis backend of the harvester:
+
+    rspec -c spec/redis_spec.rb
+    .....
+
+    Finished in 0.27042 seconds
+    5 examples, 0 failures
+
+This tests the Hash backend of the harvester:
+
+    rspec -c spec/hash_spec.rb
+    ....
+
+    Finished in 0.06071 seconds
+    4 examples, 0 failures
+
+## Questions or Problems?
+
+supercoder@gmail.com
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/barnyard_harvester.gemspec

@@ -0,0 +1,28 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "barnyard_harvester/version"
+
+Gem::Specification.new do |gem|
+  gem.name        = "barnyard_harvester"
+  gem.version     = BarnyardHarvester::VERSION
+  gem.authors     = ["Jon Gillies"]
+  gem.email       = %w(supercoder@gmail.com)
+  gem.description = %q{Performs harvests on data sources and detects adds, changes and deletes.}
+  gem.summary     = %q{Please check the README.md for more information.}
+  gem.homepage    = "https://github.com/jongillies/barnyard/tree/master/barnyard_harvester"
+
+  gem.rubyforge_project = "barnyard_harvester"
+
+  gem.files         = `git ls-files`.split("\n")
+  gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = %w(lib)
+
+  # specify any dependencies here; for example:
+  gem.add_development_dependency "rspec"
+  gem.add_runtime_dependency "resque"
+  gem.add_runtime_dependency "crack"
+  gem.add_runtime_dependency "json"
+  gem.add_runtime_dependency "uuid"
+
+end

data/lib/barnyard_harvester/hash.rb

@@ -0,0 +1,84 @@
+#!/usr/bin/env ruby
+
+require 'crack'
+require 'json'
+
+module BarnyardHarvester
+
+  class Barn
+
+    def initialize(args)
+
+      @crop_number = args.fetch(:crop_number) { raise "You must provide :crop_number" }
+      @redis_settings = args.fetch(:redis_settings) { DEFAULT_REDIS_SETTINGS }
+      @debug = args.fetch(:debug) { false }
+      @log = args.fetch(:logger) { Logger.new(STDOUT) }
+
+      @my_id = "#{args[:crop_number]}-#{self.class}"
+      @barn = Hash.new
+
+      # Setup data source connection
+      begin
+        @barn = YAML::load_file("#{@my_id}.yml")
+      rescue
+      end
+    end
+
+    def log_run(harvester_uuid, crop_number, began_at, ended_at, source_count, change_count, add_count, delete_count)
+      @log.debug "HARVESTER_LOG #{harvester_uuid}, #{crop_number}, #{began_at}, #{ended_at}, #{source_count}, #{change_count}, #{add_count}, #{delete_count})"
+    end
+
+    def delete(primary_key)
+      check_key primary_key
+      object = Crack::JSON.parse(@barn[primary_key])
+      @barn.delete primary_key
+      object # Return the object
+    end
+
+    def []= primary_key, object
+      check_key primary_key
+      check_object object
+
+      @barn[primary_key]= object.to_json
+    end
+
+    def [] primary_key
+      check_key primary_key
+
+      Crack::JSON.parse(@barn[primary_key])
+    end
+
+    def has_key?(primary_key)
+      check_key primary_key
+
+      if @barn.has_key? primary_key
+        Crack::JSON.parse(@barn[primary_key])
+      else
+        nil
+      end
+    end
+
+    def each
+      @barn.each do |primary_key, value|
+        yield primary_key, Crack::JSON.parse(value)
+      end
+    end
+
+    def flush
+      File.open("#{@my_id}.yml", "w") { |file| file.puts(@barn.to_yaml) }
+    end
+
+    private
+
+    def check_key(primary_key)
+      # Raise an exception here if the key must conform to a specific format
+      # Example: raise "key must be a string object" unless key.is_a? String
+    end
+
+    def check_object(object)
+      raise "#{object.class} must implement the to_json method" unless object.respond_to? :to_json
+    end
+  end
+
+
+end

data/lib/barnyard_harvester/hash_queue.rb

@@ -0,0 +1,53 @@
+module BarnyardHarvester
+  class Queue
+
+    def initialize(args)
+
+      @debug = args.fetch(:debug) { false }
+
+      @log = args.fetch(:logger) { Logger.new(STDOUT) }
+
+      @my_id = "#{args[:crop_number]}-#{self.class}"
+
+      @queue = Hash.new
+
+    end
+
+    def push(harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value, old_value=Hash.new)
+      check_key primary_key
+
+      @queue[primary_key] = value.to_json
+
+      @log.debug "HashQueue: Now: #{DateTime.now}, Harvester:#{harvester_uuid}, Change:#{crop_change_uuid} crop_number: #{crop_number}, key: #{primary_key}, transaction_type: #{transaction_type})"
+
+    end
+
+    def flush
+      File.open("#{@my_id}.yml", "w") { |file| file.puts(@queue.to_yaml) }
+    end
+
+    private
+
+    def check_key(primary_key)
+      # Raise an exception here if the key must conform to a specific format
+      # Example: raise "key must be a string object" unless key.is_a? String
+    end
+
+  end
+
+  # AddQueue
+  #
+  class AddQueue < Queue
+  end
+
+  # ChangeQueue
+  #
+  class ChangeQueue < Queue
+  end
+
+  # DeleteQueue
+  #
+  class DeleteQueue < Queue
+  end
+
+end

data/lib/barnyard_harvester/redis.rb

@@ -0,0 +1,93 @@
+require "redis"
+require "crack"
+require "json"
+require "resque"
+
+module BarnyardHarvester
+
+  class HarvesterLogs
+    @queue = :logs_harvester
+  end
+
+  class Barn
+
+    def initialize(args)
+
+      @crop_number = args.fetch(:crop_number) { raise "You must provide :crop_number" }
+      @redis_settings = args.fetch(:redis_settings) { DEFAULT_REDIS_SETTINGS }
+      @debug = args.fetch(:debug) { false }
+      @log = args.fetch(:logger) { Logger.new(STDOUT) }
+
+      redis_args = @redis_settings
+
+      # This sets the database number for redis to store the cached data
+      redis_args[:db] = args[:crop_number]
+
+      # Connect to Redis
+      @redis = Redis.new(redis_args)
+
+    end
+
+    def log_run(harvester_uuid, crop_number, began_at, ended_at, source_count, change_count, add_count, delete_count)
+
+      begin
+        Resque.enqueue(HarvesterLogs, Time.now, harvester_uuid, crop_number, began_at, ended_at, source_count, change_count, add_count, delete_count)
+      rescue Exception => e
+        logger.fatal "#{self.class} Fail in Resque.enqueue of HarvesterLogs. #{e.backtrace}"
+      end
+
+    end
+
+    def delete(primary_key)
+      check_key primary_key
+
+      value = @redis.get primary_key # Save the value
+      @redis.del primary_key # Delete the key
+      Crack::JSON.parse(value) # Return the object
+    end
+
+    def []= primary_key, object
+      check_key primary_key
+      check_object object
+
+      @redis.set primary_key, object.to_json
+    end
+
+    def [] primary_key
+      check_key primary_key
+
+      Crack::JSON.parse(@redis.get primary_key)
+    end
+
+    def has_key?(primary_key)
+      check_key primary_key
+
+      if @redis.exists primary_key
+        Crack::JSON.parse(@redis.get primary_key)
+      else
+        nil
+      end
+    end
+
+    def each
+      @redis.keys('*').each do |primary_key|
+        yield primary_key, Crack::JSON.parse(@redis.get(primary_key))
+      end
+    end
+
+    def flush
+    end
+
+    private
+
+    def check_key(primary_key)
+      # Raise an exception here if the key must conform to a specific format
+      # Example: raise "key must be a string object" unless key.is_a? String
+    end
+
+    def check_object(object)
+      raise "#{object.class} must implement the to_json method" unless object.respond_to? :to_json
+    end
+  end
+
+end

data/lib/barnyard_harvester/redis_queue.rb

@@ -0,0 +1,84 @@
+module BarnyardHarvester
+
+  class ChangeLogs
+    @queue = :logs_change
+  end
+
+  class Queue
+
+    class ResqueQueue
+      def initialize(queue, queued_at, harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value, old_value)
+        Resque.enqueue(queue, queued_at, harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value, old_value)
+        Resque.enqueue(ChangeLogs,queued_at, harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value, old_value)
+      end
+    end
+
+    def initialize(args)
+
+      @debug = args.fetch(:debug) { false }
+      @log = args.fetch(:logger) { Logger.new(STDOUT) }
+
+      raise "arguments must contain :crop_number => some_integer" if args[:crop_number].nil?
+
+      resque_class_name = "Distribute"
+
+      # If the class does not exist, the rescue block will create it.
+      # The Class Queue is inherited by the AddQueue, ChangeQueue and DeleteQueue, but
+      # we only want to create one "resque" queue for this instantiation
+      begin
+        Object.const_get(resque_class_name)
+      rescue
+        # Set the queue name to this apol_harvester's id prefixed with a Q_
+        #Object.const_set(resque_class_name, Class.new { @queue =  "Q_#{args[:crop_number]}"})
+        Object.const_set(resque_class_name, Class.new { @queue =  "Farmer"})
+      end
+
+      @resque_queue = Object.const_get(resque_class_name)
+
+    end
+
+    def push(harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value, old_value=Hash.new)
+      check_key primary_key
+
+      ResqueQueue.new(@resque_queue, DateTime.now, harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value.to_json, old_value.to_json)
+
+      message = "RedisQueue: #{@resque_queue}, Now: #{DateTime.now}, Harvester:#{harvester_uuid}, Change:#{crop_change_uuid} crop_number: #{crop_number}, key: #{primary_key}, transaction_type: #{transaction_type})"
+
+      if @log.level == Logger::DEBUG
+        message += ", value: #{value.to_json}, old_value: #{old_value.to_json}"
+        @log.debug message
+      end
+    end
+
+    # Flush any data if needed.
+    #
+    def flush
+    end
+
+    private
+
+    # Raise an exception here if the key must conform to a specific format
+    #
+    def check_key(primary_key)
+      # Example: raise "key must be a string object" unless key.is_a? String
+      primary_key
+    end
+
+  end
+
+  # AddQueue
+  #
+  class AddQueue < Queue
+  end
+
+  # ChangeQueue
+  #
+  class ChangeQueue < Queue
+  end
+
+  # DeleteQueue
+  #
+  class DeleteQueue < Queue
+  end
+
+end

data/lib/barnyard_harvester/version.rb

@@ -0,0 +1,3 @@
+module BarnyardHarvester
+  VERSION = "0.0.1"
+end

data/lib/barnyard_harvester.rb

@@ -0,0 +1,162 @@
+require "uuid"
+require "logger"
+
+require "barnyard_harvester/version"
+
+module BarnyardHarvester
+  ADD = "add"
+  CHANGE = "change"
+  DELETE = "delete"
+
+  # Available Options:
+  #
+  # :crop_number (required)
+  # :debug true/1
+  # :resque_enqueue true/1 (Goes to Farmer queue)
+  #
+  # :logger_call_back (calls this function with a load of params)
+  # :data_call_back (I totally forgot what this was for)
+  # :backend (default is :redis, available are: :hash, :mongodb)
+  #
+
+  DEFAULT_REDIS_SETTINGS = {:host => "localhost", :port => 6379}
+
+  class Sync
+
+    attr_reader :my_barn, :my_add_queue, :my_change_queue, :my_delete_queue
+    attr_reader :add_count, :change_count, :delete_count, :source_count, :cache_count
+    attr_reader :resque_enqueue, :redis_settings
+    attr_reader :harvester_uuid, :backend, :crop_number
+
+    def initialize(args)
+
+      @crop_number = args.fetch(:crop_number) { raise "You must provide :crop_number" }
+      @redis_settings = args.fetch(:redis_settings) { DEFAULT_REDIS_SETTINGS }
+      @debug = args.fetch(:debug) { false }
+      @log = args.fetch(:logger) { Logger.new(STDOUT) }
+
+
+      @backend = args.fetch(:backend) { :redis }
+
+      require "barnyard_harvester/#{@backend.to_s}_queue"
+      require "barnyard_harvester/#{@backend.to_s}"
+
+#      YAML::ENGINE.yamler = 'syck'
+
+      @uuid = UUID.new
+      @harvester_uuid = @uuid.generate
+
+      @key_store = Hash.new
+
+      # Setup barn and queues
+      @my_barn = BarnyardHarvester::Barn.new args
+      @my_add_queue = BarnyardHarvester::AddQueue.new args
+      @my_change_queue = BarnyardHarvester::ChangeQueue.new args
+      @my_delete_queue = BarnyardHarvester::DeleteQueue.new args
+
+      @add_count = @change_count = @delete_count = @source_count = @cache_count = 0
+
+    end
+
+    def delete_run
+
+      deletes = Array.new
+
+      # Iterate Cache Data, detect deletes.
+      @my_barn.each do |primary_key, value|
+
+        @cache_count += 1
+
+        next unless @key_store[primary_key].nil?
+
+        # We got delete
+        begin
+          crop_change_uuid = @uuid.generate
+          @my_delete_queue.push @harvester_uuid, crop_change_uuid, @crop_number, primary_key, BarnyardHarvester::DELETE, value
+        rescue Exception => e
+          @log.fatal "FATAL error pushing delete #{primary_key} to queue. #{e}"
+          exit 1
+        end
+
+        deletes << primary_key
+
+        @delete_count += 1
+      end
+
+      # Remove Deletes from the Cache Data
+      deletes.each do |primary_key|
+        @my_barn.delete primary_key
+      end
+
+    end
+
+    def process primary_key, value
+
+      @source_count += 1
+
+      crop_change_uuid = @uuid.generate
+
+      @key_store[primary_key] = :present # TODO What did this do: if @call_back.nil?
+
+      if @my_barn.has_key? primary_key
+
+        @log.debug "original: #{@my_barn[primary_key]}"
+
+        YAML::ENGINE.yamler = 'syck'
+
+        @log.debug "current : #{Crack::JSON.parse(value.to_json)}"
+
+        if @my_barn[primary_key] != Crack::JSON.parse(value.to_json)
+          #We got change!
+          begin
+            @my_change_queue.push(@harvester_uuid, crop_change_uuid, @crop_number, primary_key, BarnyardHarvester::CHANGE, value, @my_barn[primary_key])
+          rescue Exception => e
+            @log.fatal "FATAL error pushing change #{primary_key} to queue. #{e}"
+            exit 1
+          end
+
+          @my_barn[primary_key] = value
+          @change_count += 1
+        end
+      else
+        # We got add!
+        begin
+          @my_add_queue.push(@harvester_uuid, crop_change_uuid, @crop_number, primary_key, BarnyardHarvester::ADD, value)
+        rescue Exception => e
+          @log.fatal "FATAL error pushing add #{primary_key} to queue. #{e}"
+          exit 1
+        end
+
+        @my_barn[primary_key] = value
+        @add_count += 1
+      end
+    end
+
+    def stats
+      "(#{@add_count}) adds, (#{@delete_count}) deletes, (#{@change_count}) changes, (#{@source_count}) source records, (#{@cache_count}) cache records"
+    end
+
+    def run
+
+      @began_at = Time.now
+
+      yield
+
+      # Detect and queue Deletes
+      delete_run
+
+      @ended_at = Time.now
+
+      # Let Farmer know I'm done and to flush the updates
+      @my_barn.flush
+      @my_add_queue.flush
+      @my_change_queue.flush
+      @my_delete_queue.flush
+
+      @my_barn.log_run(@harvester_uuid, @crop_number, @began_at, @ended_at, @source_count, @change_count, @add_count, @delete_count)
+
+    end
+
+  end
+
+end

data/lib/test.yml

@@ -0,0 +1,15 @@
+A:
+  name: adam
+  sex: qsdasdffasdf5q345
+H:
+  name: berry
+  sex: aasdfasfdsdaasdffffffffff
+S:
+  name: cara
+  sex: asdaasdfasdf
+L:
+  name: foo
+  sex: GGGG
+Z:
+  name: foo
+  sex: GGGG

data/spec/fixtures/data-add.yml

@@ -0,0 +1,18 @@
+A:
+  name: adam
+  sex: F
+B:
+  name: berry
+  sex: M
+C:
+  name: cara
+  sex: f
+D:
+  name: dee
+  sex: f
+E:
+  name: evin
+  sex: M
+Z:
+  name: zoo
+  sex: T

data/spec/fixtures/data-change.yml

@@ -0,0 +1,15 @@
+A:
+  name: adam
+  sex: F
+B:
+  name: berry
+  sex: FFF
+C:
+  name: cara
+  sex: f
+D:
+  name: dee
+  sex: f
+E:
+  name: evin
+  sex: M

data/spec/fixtures/data-delete-all-records-add-one.yml

@@ -0,0 +1,3 @@
+P:
+  name: peter
+  sex: M

data/spec/fixtures/data-delete.yml

@@ -0,0 +1,12 @@
+A:
+  name: adam
+  sex: F
+B:
+  name: berry
+  sex: M
+C:
+  name: cara
+  sex: f
+D:
+  name: dee
+  sex: f

data/spec/fixtures/data-init.yml

@@ -0,0 +1,15 @@
+A:
+  name: adam
+  sex: F
+B:
+  name: berry
+  sex: M
+C:
+  name: cara
+  sex: f
+D:
+  name: dee
+  sex: f
+E:
+  name: evin
+  sex: M

data/spec/hash_spec.rb

@@ -0,0 +1,146 @@
+require "barnyard_harvester"
+require "yaml"
+require "redis"
+require "logger"
+require "json"
+
+
+# TODO.. Move the flush to the backend object, not here!!!!!!!!!
+
+describe BarnyardHarvester do
+
+
+  def load_and_process_file(file, backend)
+
+    data = YAML::load_file file
+
+    redis_settings = {
+        :host => "localhost",
+        :port => 6379,
+    }
+
+    my_logger = Logger.new(STDOUT)
+    my_logger.level = Logger::INFO
+
+    h = BarnyardHarvester::Sync.new(:backend => backend, :debug => false, :crop_number => 1, :hash_settings => redis_settings, :logger => my_logger)
+
+    h.run do
+      data.each do |primary_key, value|
+        h.process primary_key, value
+      end
+    end
+
+    h
+  end
+
+  def flush(crop_number)
+
+    File.delete("#{crop_number}-BarnyardHarvester::Barn.yml") if File.exist? "#{crop_number}-BarnyardHarvester::Barn.yml"
+
+  end
+
+  before(:each) do
+
+
+    @crop_number = 1
+
+    flush @crop_number
+
+    file = "spec/fixtures/data-init.yml"
+
+    data = YAML::load_file file
+
+    h = load_and_process_file(file, :hash)
+
+    h.add_count.should eq(data.count)
+    h.delete_count.should eq(0)
+    h.change_count.should eq(0)
+    h.source_count.should eq(data.count)
+    h.cache_count.should eq(data.count)
+
+  end
+
+  #it "test initial load of records" do
+  #
+  #  data = YAML::load_file "spec/fixtures/data-init.yml"
+  #
+  #  redis = Redis.new(:db => 1)
+  #
+  #  data.each do |primary_key, value|
+  #    value.to_json.should eq(redis.get(primary_key))
+  #  end
+  #
+  #end
+
+  it "test add one record" do
+
+    file = "spec/fixtures/data-add.yml"
+    data = YAML::load_file file
+
+    h = load_and_process_file(file, :hash)
+
+    h.add_count.should eq(1)
+    h.delete_count.should eq(0)
+    h.change_count.should eq(0)
+    h.source_count.should eq(data.count)
+    h.cache_count.should eq(data.count)
+
+  end
+
+  it "test change one record" do
+
+    file = "spec/fixtures/data-change.yml"
+
+    data = YAML::load_file file
+
+    h = load_and_process_file(file, :hash)
+
+    h.add_count.should eq(0)
+    h.delete_count.should eq(0)
+    h.change_count.should eq(1)
+    h.source_count.should eq(data.count)
+    h.cache_count.should eq(data.count)
+
+  end
+
+  it "test delete one record" do
+
+    file = "spec/fixtures/data-delete.yml"
+
+    data = YAML::load_file file
+
+    h = load_and_process_file(file, :hash)
+
+
+    h.add_count.should eq(0)
+    h.delete_count.should eq(1)
+    h.change_count.should eq(0)
+    h.source_count.should eq(data.count)
+    h.cache_count.should eq(data.count + 1)
+
+  end
+
+  it "test delete all records and add one" do
+
+    init_file = "spec/fixtures/data-init.yml"
+    init_data = YAML::load_file init_file
+
+    file = "spec/fixtures/data-delete-all-records-add-one.yml"
+    #data = YAML::load_file file
+
+    h = load_and_process_file(file, :hash)
+
+    h.add_count.should eq(1)
+    h.delete_count.should eq(5)
+    h.change_count.should eq(0)
+    h.source_count.should eq(1)
+    h.cache_count.should eq(init_data.count + 1)
+
+  end
+
+
+  after(:each) do
+  end
+
+
+end

data/spec/loader_spec.rb

@@ -0,0 +1,37 @@
+require "barnyard_harvester"
+
+describe BarnyardHarvester do
+
+  it "no parameters should raise error" do
+    lambda{BarnyardHarvester::Sync.new}.should raise_error
+  end
+
+  it "passing only :crop_number => 1 should return BarnyardHarvester::Sync object" do
+    BarnyardHarvester::Sync.new(crop_number: 1).class.should eq(BarnyardHarvester::Sync)
+  end
+
+  it "default backend should be :redis" do
+    BarnyardHarvester::Sync.new(crop_number: 1).backend.should eq(:redis)
+  end
+
+  it "passing backend :hash should be :hash" do
+    BarnyardHarvester::Sync.new(crop_number: 1, backend: :hash).backend.should eq(:hash)
+  end
+
+  it "passing bogus backend should raise an error" do
+    lambda{BarnyardHarvester::Sync.new(crop_number: 1, backend: :foobar)}.should raise_error
+  end
+
+  it "crop_number should be 1001" do
+    BarnyardHarvester::Sync.new(crop_number: 1001).crop_number.should eq(1001)
+  end
+
+  it "all counters should be zero" do
+    BarnyardHarvester::Sync.new(crop_number: 1).change_count.should eq(0)
+    BarnyardHarvester::Sync.new(crop_number: 1).add_count.should eq(0)
+    BarnyardHarvester::Sync.new(crop_number: 1).delete_count.should eq(0)
+    BarnyardHarvester::Sync.new(crop_number: 1).source_count.should eq(0)
+    BarnyardHarvester::Sync.new(crop_number: 1).cache_count.should eq(0)
+  end
+
+end

data/spec/redis_spec.rb

@@ -0,0 +1,149 @@
+require "barnyard_harvester"
+require "yaml"
+require "redis"
+require "logger"
+require "json"
+
+
+# TODO.. Move the flush to the backend object, not here!!!!!!!!!
+
+describe BarnyardHarvester do
+
+
+  def load_and_process_file(file, backend)
+
+    data = YAML::load_file file
+
+    redis_settings = {
+        :host => "localhost",
+        :port => 6379,
+    }
+
+    my_logger = Logger.new(STDOUT)
+    my_logger.level = Logger::INFO
+
+    h = BarnyardHarvester::Sync.new(:backend => backend, :debug => false, :crop_number => 1, :redis_settings => redis_settings, :logger => my_logger)
+
+    h.run do
+      data.each do |primary_key, value|
+        h.process primary_key, value
+      end
+    end
+
+    h
+  end
+
+    def flush
+
+      r = Redis.new(:db => 1)
+
+      r.keys.each do |k|
+        r.del k
+      end
+
+    end
+
+    before(:each) do
+
+      flush
+
+      @crop_number = 1
+
+      file = "spec/fixtures/data-init.yml"
+
+      data = YAML::load_file file
+
+      h = load_and_process_file(file, :redis)
+
+      h.add_count.should eq(data.count)
+      h.delete_count.should eq(0)
+      h.change_count.should eq(0)
+      h.source_count.should eq(data.count)
+      h.cache_count.should eq(data.count)
+
+    end
+
+    it "test initial load of records" do
+
+      data = YAML::load_file "spec/fixtures/data-init.yml"
+
+      redis = Redis.new(:db => 1)
+
+      data.each do |primary_key, value|
+        value.to_json.should eq(redis.get(primary_key))
+      end
+
+    end
+
+    it "test add one record" do
+
+      file = "spec/fixtures/data-add.yml"
+      data = YAML::load_file file
+
+      h = load_and_process_file(file, :redis)
+
+      h.add_count.should eq(1)
+      h.delete_count.should eq(0)
+      h.change_count.should eq(0)
+      h.source_count.should eq(data.count)
+      h.cache_count.should eq(data.count)
+
+    end
+
+    it "test change one record" do
+
+      file = "spec/fixtures/data-change.yml"
+
+      data = YAML::load_file file
+
+      h = load_and_process_file(file, :redis)
+
+      h.add_count.should eq(0)
+      h.delete_count.should eq(0)
+      h.change_count.should eq(1)
+      h.source_count.should eq(data.count)
+      h.cache_count.should eq(data.count)
+
+    end
+
+    it "test delete one record" do
+
+      file = "spec/fixtures/data-delete.yml"
+
+      data = YAML::load_file file
+
+      h = load_and_process_file(file, :redis)
+
+
+      h.add_count.should eq(0)
+      h.delete_count.should eq(1)
+      h.change_count.should eq(0)
+      h.source_count.should eq(data.count)
+      h.cache_count.should eq(data.count + 1)
+
+    end
+
+    it "test delete all records and add one" do
+
+      init_file = "spec/fixtures/data-init.yml"
+      init_data = YAML::load_file init_file
+
+      file = "spec/fixtures/data-delete-all-records-add-one.yml"
+      #data = YAML::load_file file
+
+      h = load_and_process_file(file, :redis)
+
+      h.add_count.should eq(1)
+      h.delete_count.should eq(5)
+      h.change_count.should eq(0)
+      h.source_count.should eq(1)
+      h.cache_count.should eq(init_data.count + 1)
+
+    end
+
+
+    after(:each) do
+    end
+
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,19 @@
+# This file was generated by the `rspec --init` command. Conventionally, all
+# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`.
+# Require this file using `require "spec_helper"` to ensure that it is only
+# loaded once.
+#
+# See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration
+RSpec.configure do |config|
+  config.treat_symbols_as_metadata_keys_with_true_values = true
+  config.run_all_when_everything_filtered = true
+  config.filter_run :focus
+
+  # Run specs in random order to surface order dependencies. If you find an
+  # order dependency and want to debug it, you can fix the order by providing
+  # the seed, which is printed after each run.
+  #     --seed 1234
+  config.order = 'random'
+end
+
+

metadata

@@ -0,0 +1,134 @@
+--- !ruby/object:Gem::Specification
+name: barnyard_harvester
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Jon Gillies
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-11-10 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &70309177353080 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70309177353080
+- !ruby/object:Gem::Dependency
+  name: resque
+  requirement: &70309177352620 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70309177352620
+- !ruby/object:Gem::Dependency
+  name: crack
+  requirement: &70309177352180 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70309177352180
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: &70309177351760 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70309177351760
+- !ruby/object:Gem::Dependency
+  name: uuid
+  requirement: &70309177351320 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70309177351320
+description: Performs harvests on data sources and detects adds, changes and deletes.
+email:
+- supercoder@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .DS_Store
+- .gitignore
+- .rvmrc
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- barnyard_harvester.gemspec
+- lib/.DS_Store
+- lib/barnyard_harvester.rb
+- lib/barnyard_harvester/hash.rb
+- lib/barnyard_harvester/hash_queue.rb
+- lib/barnyard_harvester/redis.rb
+- lib/barnyard_harvester/redis_queue.rb
+- lib/barnyard_harvester/version.rb
+- lib/test.yml
+- spec/fixtures/data-add.yml
+- spec/fixtures/data-change.yml
+- spec/fixtures/data-delete-all-records-add-one.yml
+- spec/fixtures/data-delete.yml
+- spec/fixtures/data-init.yml
+- spec/hash_spec.rb
+- spec/loader_spec.rb
+- spec/redis_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/jongillies/barnyard/tree/master/barnyard_harvester
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: barnyard_harvester
+rubygems_version: 1.8.15
+signing_key: 
+specification_version: 3
+summary: Please check the README.md for more information.
+test_files:
+- spec/fixtures/data-add.yml
+- spec/fixtures/data-change.yml
+- spec/fixtures/data-delete-all-records-add-one.yml
+- spec/fixtures/data-delete.yml
+- spec/fixtures/data-init.yml
+- spec/hash_spec.rb
+- spec/loader_spec.rb
+- spec/redis_spec.rb
+- spec/spec_helper.rb