barnyard_harvester 0.0.3 → 0.0.5

data/barnyard_harvester.gemspec

@@ -24,5 +24,6 @@ Gem::Specification.new do |gem|
   gem.add_runtime_dependency "crack"
   gem.add_runtime_dependency "json"
   gem.add_runtime_dependency "uuid"
-
+  gem.add_runtime_dependency "bson_ext", "= 1.6.0"
+  gem.add_runtime_dependency "mongo", "= 1.6.0"
 end

data/lib/barnyard_harvester.rb

@@ -14,7 +14,7 @@ module BarnyardHarvester
 
     attr_reader :my_barn, :my_add_queue, :my_change_queue, :my_delete_queue
     attr_reader :add_count, :change_count, :delete_count, :source_count, :cache_count
-    attr_reader :resque_enqueue, :redis_settings
+    attr_reader :redis_settings
     attr_reader :harvester_uuid, :backend, :crop_number
 
     def initialize(args)
@@ -27,6 +27,11 @@ module BarnyardHarvester
 
       @backend = args.fetch(:backend) { :redis }
 
+      if @backend == :mongodb
+        @mongodb_settings = args.fetch(:mongodb_settings) { raise "You must provide :mongodb_settings" }
+      end
+
+      require "barnyard_harvester/#{@backend.to_s}_helper" if File.exist? "barnyard_harvester/#{@backend.to_s}_helper"
       require "barnyard_harvester/#{@backend.to_s}_queue"
       require "barnyard_harvester/#{@backend.to_s}"
 

data/lib/barnyard_harvester/mongodb.rb

@@ -0,0 +1,141 @@
+require "mongo"
+require "crack"
+require "json"
+require "resque"
+require "barnyard_harvester/mongodb_helper"
+
+#
+#require "lib/barnyard_harvester/mongodb_helper"
+
+module BarnyardHarvester
+
+  class HarvesterLogs
+    @queue = :logs_harvester
+  end
+
+  DEFAULT_MONGO_SETTINGS = {
+      :host_list => "localhost:27017",
+      :collection => "test_collection",
+      :db => "test_db"
+  }
+
+  class Barn
+
+    def initialize(args)
+
+      @crop_number = args.fetch(:crop_number) { raise "You must provide :crop_number" }
+      @redis_settings = args.fetch(:redis_settings) { DEFAULT_REDIS_SETTINGS }
+      @mongodb_settings = args.fetch(:mongodb_settings) { MongoSettings }
+      @debug = args.fetch(:debug) { false }
+      @log = args.fetch(:logger) { Logger.new(STDOUT) }
+
+      @mongodb_settings.fetch(:db) { raise "You must provide :db" }
+      @mongodb_settings.fetch(:collection) { raise "You must provide :collection" }
+
+      @redis_settings.delete(:db)
+      Resque.redis = Redis.new(@redis_settings)
+
+      @mongodb_settings[:debug] = @debug
+      @mongodb_settings[:logger] = @log
+
+      @mongo = BarnyardHarvester::MongoDbHelper.connect @mongodb_settings
+
+      @collection_name = @mongodb_settings[:collection]
+      @collection = @mongo.collection(@mongodb_settings[:collection])
+
+      @collection
+    end
+
+    def log_run(harvester_uuid, crop_number, began_at, ended_at, source_count, change_count, add_count, delete_count)
+
+      begin
+        Resque.enqueue(HarvesterLogs, Time.now, harvester_uuid, crop_number, began_at, ended_at, source_count, change_count, add_count, delete_count)
+      rescue Exception => e
+        logger.fatal "#{self.class} Fail in Resque.enqueue of HarvesterLogs. #{e.backtrace}"
+      end
+
+    end
+
+    def delete(primary_key)
+      check_key primary_key
+
+      value = @collection.find "_id" => primary_key # Save the value
+
+      @collection.remove("_id" => primary_key)
+
+      value.to_json # Return the object
+    end
+
+    def []= primary_key, object
+      check_key primary_key
+      check_object object
+
+      obj = Crack::JSON.parse object.to_json
+
+      # We artificially add the _id value to the object as this is the primary key
+      # This is stored in the Mongo database, but removed upon a fetch.
+
+      if obj.is_a?(Hash)
+        obj["_id"] = primary_key
+      else
+        @log.fatal "WOAH! Class: #{object.class} Value: #{object}"
+        return
+      end
+
+
+      if self.has_key? primary_key
+        @collection.update({"_id" => primary_key}, obj)
+      else
+        @collection.insert obj
+      end
+
+    end
+
+    def [] primary_key
+      check_key primary_key
+
+      doc = @collection.find("_id" => primary_key).to_a[0]
+
+      #
+      # Delete the "_id" from the document as it is artificial for the MongoDB primary key
+      doc.delete("_id")
+
+      Crack::JSON.parse(doc.to_json)
+    end
+
+    def has_key?(primary_key)
+      check_key primary_key
+
+      doc = @collection.find "_id" => primary_key
+      if  doc.count == 0
+        false
+      else
+        true
+      end
+    end
+
+    def each
+
+      collection = @mongo.collection(@collection_name)
+
+      collection.find.each do |row|
+        yield row["_id"], row.to_json
+      end
+
+    end
+
+    def flush
+    end
+
+    private
+
+    def check_key(primary_key)
+      # Raise an exception here if the key must conform to a specific format
+      # Example: raise "key must be a string object" unless key.is_a? String
+    end
+
+    def check_object(object)
+      raise "#{object.class} must implement the to_json method" unless object.respond_to? :to_json
+    end
+  end
+end

data/lib/barnyard_harvester/mongodb_helper.rb

@@ -0,0 +1,72 @@
+require "mongo"
+require "logger"
+
+module BarnyardHarvester
+  class MongoDbHelper
+
+    def self.connect(args)
+
+      @debug = args.fetch(:debug) { false }
+      @log = args.fetch(:logger) { Logger.new(STDOUT) }
+
+      @log.debug "Connection parameters #{args[:host]} db: #{args[:db]} collection: #{args[:collection]}" if @debug
+
+      # Connect to Mongo
+      if args.has_key? :host
+        if args[:host].is_a? Array
+          @log.debug "Connecting to replica set #{args[:host]}"
+          @mongo = Mongo::ReplSetConnection.new(args[:host]).db(args[:db])
+        else
+          @log.debug "Connecting to single host #{args[:host]}"
+          @mongo = Mongo::Connection.new(args[:host].split(":")[0], args[:host].split(":")[1]).db(args[:db])
+        end
+      else
+        @log.info "Connecting to localhost #{args[:host]}"
+        @mongo = Mongo::Connection.new.db(args[:db])
+      end
+
+      #db = @mongo.db((mongo_args[:db])
+      #auth = db.authenticate
+
+      unless args[:user].to_s == ''
+        @log.debug "Authenticating as #{args[:user]}"
+        @mongo.authenticate(args[:user], args[:password])
+      end
+
+      @mongo
+    end
+  end
+end
+
+if __FILE__ == $0
+
+  mongo_replica_set = Array.new
+
+  mongo_replica_set << "ip-172-19-31-44.c.qaapollogrp.edu:27017"
+  mongo_replica_set << "ip-172-19-30-49.c.qaapollogrp.edu:27017"
+  mongo_replica_set << "ip-172-19-31-202.c.qaapollogrp.edu:27017"
+
+  mongo_args = Hash.new
+#  mongo_args[:host] = "localhost:27017"
+  mongo_args[:host] = mongo_replica_set
+  mongo_args[:db] = "aws"
+  mongo_args[:collection] = "test_collection"
+  mongo_args[:user] = "honeybadger"
+  mongo_args[:password] = "0joQuk35vJM05Hj"
+
+  mongo_args[:debug] = true
+
+  my_log = Logger.new(STDOUT)
+
+  mongo_args[:logger] = my_log
+
+  s = ApolMongo::MongoHelper.connect mongo_args
+
+  c = s.collection(mongo_args[:collection])
+
+
+  c.find.each do |row|
+    puts row
+  end
+
+end

data/lib/barnyard_harvester/mongodb_queue.rb

@@ -0,0 +1,84 @@
+module BarnyardHarvester
+
+  class ChangeLogs
+    @queue = :logs_change
+  end
+
+  class Queue
+
+    class ResqueQueue
+      def initialize(queue, queued_at, harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value, old_value)
+        Resque.enqueue(queue, queued_at, harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value, old_value)
+        Resque.enqueue(ChangeLogs,queued_at, harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value, old_value)
+      end
+    end
+
+    def initialize(args)
+
+      @debug = args.fetch(:debug) { false }
+      @log = args.fetch(:logger) { Logger.new(STDOUT) }
+
+      raise "arguments must contain :crop_number => some_integer" if args[:crop_number].nil?
+
+      resque_class_name = "Distribute"
+
+      # If the class does not exist, the rescue block will create it.
+      # The Class Queue is inherited by the AddQueue, ChangeQueue and DeleteQueue, but
+      # we only want to create one "resque" queue for this instantiation
+      begin
+        Object.const_get(resque_class_name)
+      rescue
+        # Set the queue name to this apol_harvester's id prefixed with a Q_
+        #Object.const_set(resque_class_name, Class.new { @queue =  "Q_#{args[:crop_number]}"})
+        Object.const_set(resque_class_name, Class.new { @queue =  "Farmer"})
+      end
+
+      @resque_queue = Object.const_get(resque_class_name)
+
+    end
+
+    def push(harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value, old_value=Hash.new)
+      check_key primary_key
+
+      ResqueQueue.new(@resque_queue, DateTime.now, harvester_uuid, crop_change_uuid, crop_number, primary_key, transaction_type, value.to_json, old_value.to_json)
+
+      message = "RedisQueue: #{@resque_queue}, Now: #{DateTime.now}, Harvester:#{harvester_uuid}, Change:#{crop_change_uuid} crop_number: #{crop_number}, key: #{primary_key}, transaction_type: #{transaction_type})"
+
+      if @log.level == Logger::DEBUG
+        message += ", value: #{value.to_json}, old_value: #{old_value.to_json}"
+        @log.debug message
+      end
+    end
+
+    # Flush any data if needed.
+    #
+    def flush
+    end
+
+    private
+
+    # Raise an exception here if the key must conform to a specific format
+    #
+    def check_key(primary_key)
+      # Example: raise "key must be a string object" unless key.is_a? String
+      primary_key
+    end
+
+  end
+
+  # AddQueue
+  #
+  class AddQueue < Queue
+  end
+
+  # ChangeQueue
+  #
+  class ChangeQueue < Queue
+  end
+
+  # DeleteQueue
+  #
+  class DeleteQueue < Queue
+  end
+
+end

data/lib/barnyard_harvester/version.rb

@@ -1,3 +1,3 @@
 module BarnyardHarvester
-  VERSION = "0.0.3"
+  VERSION = "0.0.5"
 end

data/spec/mongo_helper_spec.rb

@@ -0,0 +1,61 @@
+require "rspec"
+require "./lib/barnyard_harvester/mongodb_helper"
+
+describe BarnyardHarvester::MongoDbHelper do
+
+  it "should connect to a replica set" do
+
+    mongo_replica_set = Array.new
+
+    mongo_replica_set << "ip-172-19-31-44.c.qaapollogrp.edu:27017"
+    mongo_replica_set << "ip-172-19-30-49.c.qaapollogrp.edu:27017"
+    mongo_replica_set << "ip-172-19-31-202.c.qaapollogrp.edu:27017"
+    mongo_args = Hash.new
+
+
+    mongo_args[:host] = mongo_replica_set
+    mongo_args[:db] = "aws"
+    mongo_args[:collection] = "test_collection"
+    mongo_args[:user] = "honeybadger"
+    mongo_args[:password] = "0joQuk35vJM05Hj"
+
+    mongo_args[:debug] = true
+
+    my_log = Logger.new(STDOUT)
+
+    mongo_args[:logger] = my_log
+
+    s = BarnyardHarvester::MongoDbHelper.connect mongo_args
+
+    c = s.collection(mongo_args[:collection])
+
+    c.find.each do |row|
+      puts row
+    end
+
+  end
+
+  it "should connect to one server" do
+
+    mongo_args = Hash.new
+    mongo_args[:host] = "localhost:27017"
+    mongo_args[:db] = "aws"
+    mongo_args[:collection] = "test_collection"
+
+    mongo_args[:debug] = true
+
+    my_log = Logger.new(STDOUT)
+
+    mongo_args[:logger] = my_log
+
+    s = BarnyardHarvester::MongoDbHelper.connect mongo_args
+
+    c = s.collection(mongo_args[:collection])
+
+    c.find.each do |row|
+      puts row
+    end
+
+  end
+
+end

data/spec/mongo_spec.rb

@@ -0,0 +1,180 @@
+require "barnyard_harvester"
+require "yaml"
+require "mongo"
+require "redis"
+require "logger"
+require "json"
+
+CROP_NUMBER = 1
+
+REDIS_SETTINGS = {
+    :host => "localhost",
+    :port => 6379,
+    :db => CROP_NUMBER
+}
+
+MONGODB_SETTINGS = {
+    :host => "localhost",
+    :collection => "test_collection",
+    :db => "test_database",
+    :collection => "test_collection"
+}
+
+MONGODB_REPLICA_SET_SETTINGS = {
+    :host => ["ip-172-19-31-44.c.qaapollogrp.edu:27017", "ip-172-19-30-49.c.qaapollogrp.edu:27017", "ip-172-19-31-202.c.qaapollogrp.edu:27017"],
+    :collection => "test_collection",
+    :db => "test_database",
+    :user => "honeybadger",
+    :password => "0joQuk35vJM05Hj",
+    :collection => "test_collection"
+}
+
+$mongo_settings = MONGODB_REPLICA_SET_SETTINGS
+
+describe BarnyardHarvester do
+
+  def load_and_process_file(file, backend)
+
+    data = YAML::load_file file
+
+    my_logger = Logger.new(STDOUT)
+    my_logger.level = Logger::INFO
+
+    h = BarnyardHarvester::Sync.new(:backend => backend, :debug => false, :mongodb_settings => $mongo_settings, :crop_number => CROP_NUMBER, :redis_settings => REDIS_SETTINGS, :logger => my_logger)
+
+    h.run do
+      data.each do |primary_key, value|
+        h.process primary_key, value
+      end
+    end
+
+    h
+  end
+
+  def flush
+
+    require "barnyard_harvester/mongodb_helper"
+
+    my_logger = Logger.new(STDOUT)
+    my_logger.level = Logger::INFO
+
+    mongo = BarnyardHarvester::MongoDbHelper.connect $mongo_settings.merge(logger: my_logger)
+    collection_name = $mongo_settings[:collection]
+    collection = mongo.collection(collection_name)
+
+    collection.find.each do |row|
+      collection.remove("_id" => row["_id"])
+    end
+
+  end
+
+  before(:each) do
+
+    flush
+
+    @crop_number = 1
+
+    file = "spec/fixtures/data-init.yml"
+
+    data = YAML::load_file file
+
+    h = load_and_process_file(file, :mongodb)
+
+    h.add_count.should eq(data.count)
+    h.delete_count.should eq(0)
+    h.change_count.should eq(0)
+    h.source_count.should eq(data.count)
+    h.cache_count.should eq(data.count)
+
+  end
+
+  it "test initial load of records" do
+
+    data = YAML::load_file "spec/fixtures/data-init.yml"
+
+    mongo = BarnyardHarvester::MongoDbHelper.connect $mongo_settings
+    collection = mongo.collection($mongo_settings[:collection])
+
+    data.each do |primary_key, value|
+
+      doc = collection.find("_id" => primary_key).to_a[0]
+
+      doc.delete("_id")
+
+      value.should eq(doc)
+    end
+
+  end
+
+  it "test add one record" do
+
+    file = "spec/fixtures/data-add.yml"
+    data = YAML::load_file file
+
+    h = load_and_process_file(file, :mongodb)
+
+    h.add_count.should eq(1)
+    h.delete_count.should eq(0)
+    h.change_count.should eq(0)
+    h.source_count.should eq(data.count)
+    h.cache_count.should eq(data.count)
+
+  end
+
+  it "test change one record" do
+
+    file = "spec/fixtures/data-change.yml"
+
+    data = YAML::load_file file
+
+    h = load_and_process_file(file, :mongodb)
+
+    h.add_count.should eq(0)
+    h.delete_count.should eq(0)
+    h.change_count.should eq(1)
+    h.source_count.should eq(data.count)
+    h.cache_count.should eq(data.count)
+
+  end
+
+  it "test delete one record" do
+
+    file = "spec/fixtures/data-delete.yml"
+
+    data = YAML::load_file file
+
+    h = load_and_process_file(file, :mongodb)
+
+
+    h.add_count.should eq(0)
+    h.delete_count.should eq(1)
+    h.change_count.should eq(0)
+    h.source_count.should eq(data.count)
+    h.cache_count.should eq(data.count + 1)
+
+  end
+
+  it "test delete all records and add one" do
+
+    init_file = "spec/fixtures/data-init.yml"
+    init_data = YAML::load_file init_file
+
+    file = "spec/fixtures/data-delete-all-records-add-one.yml"
+    #data = YAML::load_file file
+
+    h = load_and_process_file(file, :mongodb)
+
+    h.add_count.should eq(1)
+    h.delete_count.should eq(5)
+    h.change_count.should eq(0)
+    h.source_count.should eq(1)
+    h.cache_count.should eq(init_data.count + 1)
+
+  end
+
+
+  after(:each) do
+  end
+
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: barnyard_harvester
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.5
   prerelease: 
 platform: ruby
 authors:
@@ -9,11 +9,11 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-11-12 00:00:00.000000000 Z
+date: 2012-11-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &70154684272920 !ruby/object:Gem::Requirement
+  requirement: &70118633756900 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -21,10 +21,10 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *70154684272920
+  version_requirements: *70118633756900
 - !ruby/object:Gem::Dependency
   name: resque
-  requirement: &70154684271860 !ruby/object:Gem::Requirement
+  requirement: &70118633753220 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -32,10 +32,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70154684271860
+  version_requirements: *70118633753220
 - !ruby/object:Gem::Dependency
   name: crack
-  requirement: &70154684271380 !ruby/object:Gem::Requirement
+  requirement: &70118633748400 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -43,10 +43,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70154684271380
+  version_requirements: *70118633748400
 - !ruby/object:Gem::Dependency
   name: json
-  requirement: &70154684270700 !ruby/object:Gem::Requirement
+  requirement: &70118633735240 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -54,10 +54,10 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70154684270700
+  version_requirements: *70118633735240
 - !ruby/object:Gem::Dependency
   name: uuid
-  requirement: &70154684270020 !ruby/object:Gem::Requirement
+  requirement: &70118633729920 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -65,7 +65,29 @@ dependencies:
         version: '0'
   type: :runtime
   prerelease: false
-  version_requirements: *70154684270020
+  version_requirements: *70118633729920
+- !ruby/object:Gem::Dependency
+  name: bson_ext
+  requirement: &70118633723020 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - =
+      - !ruby/object:Gem::Version
+        version: 1.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *70118633723020
+- !ruby/object:Gem::Dependency
+  name: mongo
+  requirement: &70118633718980 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - =
+      - !ruby/object:Gem::Version
+        version: 1.6.0
+  type: :runtime
+  prerelease: false
+  version_requirements: *70118633718980
 description: Performs harvests on data sources and detects adds, changes and deletes.
 email:
 - supercoder@gmail.com
@@ -85,6 +107,9 @@ files:
 - lib/barnyard_harvester.rb
 - lib/barnyard_harvester/hash.rb
 - lib/barnyard_harvester/hash_queue.rb
+- lib/barnyard_harvester/mongodb.rb
+- lib/barnyard_harvester/mongodb_helper.rb
+- lib/barnyard_harvester/mongodb_queue.rb
 - lib/barnyard_harvester/redis.rb
 - lib/barnyard_harvester/redis_queue.rb
 - lib/barnyard_harvester/version.rb
@@ -96,6 +121,8 @@ files:
 - spec/fixtures/data-init.yml
 - spec/hash_spec.rb
 - spec/loader_spec.rb
+- spec/mongo_helper_spec.rb
+- spec/mongo_spec.rb
 - spec/redis_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/jongillies/barnyard/tree/master/barnyard_harvester
@@ -118,7 +145,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: barnyard_harvester
-rubygems_version: 1.8.11
+rubygems_version: 1.8.15
 signing_key: 
 specification_version: 3
 summary: Please check the README.md for more information.
@@ -130,5 +157,7 @@ test_files:
 - spec/fixtures/data-init.yml
 - spec/hash_spec.rb
 - spec/loader_spec.rb
+- spec/mongo_helper_spec.rb
+- spec/mongo_spec.rb
 - spec/redis_spec.rb
 - spec/spec_helper.rb