dca 0.1.0

data/lib/dca/notifier/redis/notifier.rb

@@ -0,0 +1,25 @@
+module DCA
+  module Redis
+    class Notifier
+      def initialize config
+        Ohm.connect :url => "redis://#{config[:host]}:#{config[:port]}"
+      end
+
+      def push(object, event, options = {})
+        session = Session.find(:uid => object.session, :project => DCA.project_name, :area => object.class.queue).first
+        unless session.present?
+          session = Session.create :uid => object.session, :created => Time.now, :project => DCA.project_name,
+                                   :area => object.class.queue
+        end
+
+        if event == :analyze
+          session.inc_analyze options[:state]
+        elsif event == :fetch
+          session.inc_fetch options[:state], options[:result] ? :success : :failure
+        elsif event == :failure
+          session.add_failure options[:exception]
+        end
+      end
+    end
+  end
+end

data/lib/dca/notifier/redis_notifier.rb

@@ -0,0 +1,9 @@
+require File.expand_path('../redis/models/failure_notify', __FILE__)
+require File.expand_path('../redis/models/analyze_notify', __FILE__)
+require File.expand_path('../redis/models/fetch_notify', __FILE__)
+require File.expand_path('../redis/models/session', __FILE__)
+require File.expand_path('../redis/notifier', __FILE__)
+
+module DCA
+  class RedisNotifier < Redis::Notifier; end
+end

data/lib/dca/storage.rb

@@ -0,0 +1,3 @@
+require File.expand_path('../storage/storage', __FILE__)
+require File.expand_path('../storage/elasticsearch_storage', __FILE__)
+require File.expand_path('../storage/mongo_storage', __FILE__)

data/lib/dca/storage/elasticsearch_storage.rb

@@ -0,0 +1,80 @@
+module DCA
+  class ElasticSearchStorage
+    attr_reader :type
+
+    def initialize(connection, context, options = {})
+      @connection = connection
+      @index = options[:index] || DCA.project_name.underscore
+      @type = options[:type] || context.to_s.demodulize.downcase.pluralize
+    end
+
+    def self.establish_connection(config)
+      RestClient.get("http://#{config[:host]}:#{config[:port]}")
+      Tire.configure { url "http://#{config[:host]}:#{config[:port]}" }
+    end
+
+
+    def state position
+      item = find position
+      return :create if item.nil?
+
+      position.id = item[:id]
+      return :unmodified if item[:checksum] == position.checksum
+
+      :update
+    end
+
+    def find position
+      return nil if position.base_id.nil?
+      query = Tire.search(@index, :type => type) { query { term :base_id, position.base_id } }
+      query.results.first
+    end
+
+    def refresh(item, state)
+      send state, item
+    end
+
+    def create(item)
+      item.updated_at = item.created_at = Time.now.utc
+      data = item.to_hash
+      data[:type] = type
+
+      result = Tire.index(@index).store data
+      Tire.index(@index).refresh
+
+      item.id = result['_id']
+    end
+
+    def update(item)
+      data = item.to_hash
+      data[:type] = type
+
+      Tire.index(@index) do
+        store data
+        refresh
+      end
+    end
+
+    def remove(item)
+      data = item.to_hash
+      data[:type] = type
+
+      Tire.index(@index) do
+        remove data
+        refresh
+      end
+    end
+
+    def index(&block)
+      Tire.index @index do
+        instance_eval(&block) if block_given?
+      end
+    end
+
+    def context object
+      result = self.clone
+      result.instance_variable_set :@type, object.to_s.demodulize.downcase.pluralize
+      result
+    end
+  end
+end

data/lib/dca/storage/mongo_storage.rb

@@ -0,0 +1,51 @@
+module DCA
+  class MongoStorage
+    attr_reader :database, :collection
+
+    def initialize(connection, context, options = {})
+      @database = connection.db(options[:database] || DCA.project_name.underscore)
+      @connection = connection
+      @collection = database.collection(options[:collection] || context.to_s.demodulize.downcase.pluralize)
+    end
+
+    def self.establish_connection(config)
+      Mongo::Connection.new config[:host], config[:port]
+    end
+
+    def state(position)
+      item = find position
+      return :create if item.nil?
+
+      position.id = item['_id']
+      return :unmodified if item['checksum'] == position.checksum
+
+      return :update
+    end
+
+    def find position
+      collection.find_one base_id: position.base_id unless position.base_id.nil?
+    end
+
+    def refresh(item, state)
+      send state, item
+    end
+
+    def create(item)
+      item.id = collection.insert item.to_hash
+    end
+
+    def update(item)
+      collection.update({_id: item.id}, item.to_hash)
+    end
+
+    def remove(item)
+      collection.remove _id: item.id
+    end
+
+    def context object
+      result = self.clone
+      result.instance_variable_set :@collection, result.database.collection(object.to_s.demodulize.downcase.pluralize)
+      result
+    end
+  end
+end

data/lib/dca/storage/storage.rb

@@ -0,0 +1,55 @@
+module DCA
+  module Storage
+    extend ActiveSupport::Concern
+
+    module ClassMethods
+      def establish_connection config_name = :db
+        @config = APP_CONFIG[config_name]
+      end
+
+      def root_storage
+        return @storage if @storage
+
+        return storage if @config
+
+        superclass.root_storage if superclass != Object
+      end
+
+      def storage
+        return @storage unless @storage.nil?
+
+        if @config
+          driver_class = "DCA::#{@config[:driver]}Storage".constantize
+          connection = driver_class.establish_connection @config
+          @storage = driver_class.new connection, self, @config
+        else
+          @storage = superclass.root_storage.context self
+        end
+      end
+    end
+
+    def state
+      self.class.storage.state self
+    end
+
+    def save
+      return false unless valid?
+
+      current_state = state
+
+      callback = "before_#{state}"
+      send callback if self.respond_to? callback
+
+      self.class.storage.refresh self, current_state
+
+      callback = "after_#{state}"
+      send callback if self.respond_to? callback
+
+      current_state
+    end
+
+    def destroy
+      self.class.storage.refresh self, :remove
+    end
+  end
+end

data/spec/analyzer_spec.rb

@@ -0,0 +1,64 @@
+require File.expand_path('../spec_helper', __FILE__)
+require File.expand_path('../mock/position', __FILE__)
+require File.expand_path('../mock/analyzer_job', __FILE__)
+require File.expand_path('../mock/file_storage', __FILE__)
+require File.expand_path('../mock/web_notifier', __FILE__)
+
+include DCA
+
+describe 'Analyzer jobs' do
+  before :all do
+    Notifier.create APP_CONFIG[:notifier]
+  end
+
+  after :all do
+    Tire.index('test') do
+      delete
+      refresh
+    end
+  end
+
+  it "should notice about analyzed positions" do
+    Mock::WebNotifier.clean
+    Mock::AnalyzerJob.create :fixture => './spec/fixtures/positions.yml'
+    Mock::WebNotifier.queue[:analyze][:create].should equal 2
+  end
+
+  it "should notice about fetch positions" do
+    Mock::WebNotifier.clean
+    Mock::AnalyzerJob.create :fixture => './spec/fixtures/positions.yml'
+    Mock::WebNotifier.queue[:fetch][:create].should equal 2
+  end
+
+  it "should notice about failed fetch positions" do
+    Mock::WebNotifier.clean
+    Mock::AnalyzerJob.create :fixture => './spec/fixtures/positions.yml'
+    Mock::WebNotifier.queue[:failed][:create].should equal 1
+  end
+
+  it "should fetch only newly created and modified position" do
+    Mock::WebNotifier.clean
+    Mock::AnalyzerJob.create :fixture => './spec/fixtures/positions.yml'
+    Mock::WebNotifier.queue[:analyze][:unmodified].should equal 1
+    Mock::WebNotifier.queue[:fetch][:create].should equal 2
+  end
+
+  it "should fetch next position even if exception occurs" do
+    Mock::WebNotifier.clean
+    Mock::AnalyzerJob.create :fixture => './spec/fixtures/positions_with_error.yml'
+    Mock::WebNotifier.queue[:failed][:create].should equal 1
+  end
+
+  it "should analyzed only limit count of posiotions" do
+    Mock::WebNotifier.clean
+    Mock::AnalyzerJob.create :limit => 1, :fixture => './spec/fixtures/positions.yml'
+    Mock::WebNotifier.queue[:analyze][:create].should equal 1
+  end
+
+  it "should support distribute analyze" do
+    Mock::WebNotifier.clean
+    Mock::AnalyzerJob.create :fixture => './spec/fixtures/positions.yml', :distributed => true
+    Mock::WebNotifier.queue[:analyze][:unmodified].should equal 1
+    Mock::WebNotifier.queue[:fetch][:create].should equal 2
+  end
+end

data/spec/area_task_spec.rb

@@ -0,0 +1,45 @@
+require File.expand_path('../spec_helper', __FILE__)
+require File.expand_path('../mock/analyzer_job', __FILE__)
+
+describe 'Area rake task' do
+  def workers_clean
+    `ps aux | grep -P "resque-\\d"`.split("\n").each do |line|
+      pid = line.split(' ')[1].to_i
+      Process.kill 'TERM', pid
+    end
+
+    sleep 1 while workers_count > 0
+  end
+
+  before :each do
+    Resque.inline = false
+    workers_clean
+  end
+
+  after :all do
+    Resque.remove_queue 'Mock'
+    Resque.inline = true
+    workers_clean
+    #FileUtils.rm_rf File.join(DCA.root, 'log')
+  end
+
+  def workers_count
+    `ps aux | grep -P "resque-\\d"`.split("\n").count
+  end
+
+  it 'should stop area analyze work' do
+    current_count = workers_count + 1
+    `rake resque:work BACKGROUND=1 QUEUE=Mock`
+    sleep 1
+    workers_count.should equal current_count
+
+    DCA::CLI.new.area 'stop', 'Mock'
+    workers_count.should equal 0
+  end
+
+  it 'should start area analyze work' do
+    DCA::CLI.new.area 'start', 'Mock'
+    sleep 2
+    workers_count.should equal 1
+  end
+end

data/spec/base_model_spec.rb

@@ -0,0 +1,34 @@
+require File.expand_path('../spec_helper', __FILE__)
+require File.expand_path('../mock/position', __FILE__)
+
+include DCA
+
+describe Models::BaseModel do
+  let(:position) { Mock::RootPosition.new }
+  let(:child_position) { Mock::ChildPosition.new }
+  describe 'when validate has_one associations and model invalid' do
+    before do
+      position.one_child = child_position
+      position.valid?
+    end
+
+    it 'should contains associations errors' do
+      position.errors[:one_child].should_not be_nil
+    end
+  end
+
+  describe 'when validate has_many associations and model invalid' do
+    before do
+      position.child_position = [child_position, child_position]
+      position.valid?
+    end
+
+    it 'should contains associations errors' do
+      position.errors[:child_position].should_not be_nil
+    end
+
+    it 'should contains associations errors array' do
+      position.errors[:child_position].count.should equal 2
+    end
+  end
+end

data/spec/binder_spec.rb

@@ -0,0 +1,69 @@
+require File.expand_path('../spec_helper', __FILE__)
+require File.expand_path('../mock/position', __FILE__)
+require File.expand_path('../mock/page', __FILE__)
+
+include DCA
+
+describe 'Binder' do
+  it 'should bind model fields' do
+    page = Mock::Page.new
+    page.bind Nokogiri::HTML open('./spec/fixtures/page.html')
+
+    page.positions.should have(3).items
+    page.positions.each_with_index do |position, index|
+      position.base_id.should == index + 1
+      position.title.should == "Position #{index + 1}"
+    end
+  end
+
+  it 'should bind polymorphic association' do
+    page = Mock::PageExt.new :category => :full
+    page.bind Nokogiri::HTML open('./spec/fixtures/page.html')
+
+    page.positions.should have(3).items
+    page.positions.each { |position| position.is_a?(Mock::FullPosition).should be_true }
+
+
+    page = Mock::PageExt.new :category => :ext
+    page.bind Nokogiri::HTML open('./spec/fixtures/page.html')
+
+    page.positions.should have(3).items
+    page.positions.each { |position| position.is_a?(Mock::ExtPosition).should be_true }
+  end
+
+  it 'should inherit binding options' do
+    Mock::ExtPosition.associations.should have(4).items
+    Mock::ExtPosition.associations.keys.should include(:base_id, :title, :description)
+  end
+
+  it 'should override binding options' do
+    Mock::FullPosition.associations.should have(2).items
+    Mock::FullPosition.associations[:base_id][:type].should == :string
+    Mock::FullPosition.associations[:base_id][:options].should == { :selector => 'a' }
+  end
+
+  it 'should serializable' do
+    position = Mock::ChildPosition.new name: 'position'
+    position.to_hash.should have_key 'name'
+  end
+
+  it 'should serializable with child' do
+    position = Mock::RootPosition.new
+    position.one_child = Mock::ChildPosition.new(name: 'child_position')
+    position.child_position = [Mock::ChildPosition.new(name: 'position')]
+
+    hash = position.to_hash
+    hash['one_child'].is_a?(Hash).should == true
+
+    hash['child_position'][0].is_a?(Hash).should == true
+  end
+
+  it 'should append to has many association' do
+    page = Mock::PageExt.new :category => :full
+    page.bind Nokogiri::HTML open('./spec/fixtures/page.html')
+    page.bind Nokogiri::HTML open('./spec/fixtures/page.html')
+
+    page.positions.should have(6).items
+
+  end
+end