RubyGems - dca - Versions diffs - 0.1.0 - Mend

dca 0.1.0

Files changed (70) hide show

data/.document +5 -0
data/.rspec +2 -0
data/Gemfile +48 -0
data/Gemfile.lock +126 -0
data/Guardfile +8 -0
data/LICENSE.txt +20 -0
data/README.rdoc +19 -0
data/Rakefile +48 -0
data/VERSION +1 -0
data/bin/dca +5 -0
data/dca.gemspec +160 -0
data/lib/dca.rb +64 -0
data/lib/dca/cli.rb +32 -0
data/lib/dca/commands/area.rb +133 -0
data/lib/dca/commands/templates/area/analyzer.rb.erb +34 -0
data/lib/dca/commands/templates/area/area.rb.erb +2 -0
data/lib/dca/commands/templates/area/models.rb.erb +2 -0
data/lib/dca/commands/templates/area/page.rb.erb +17 -0
data/lib/dca/commands/templates/area/position.rb.erb +8 -0
data/lib/dca/commands/templates/config.yml.erb +38 -0
data/lib/dca/commands/templates/spec/analyzer_spec.rb.erb +15 -0
data/lib/dca/commands/templates/spec/spec_helper.rb.erb +2 -0
data/lib/dca/config.rb +20 -0
data/lib/dca/helpers.rb +2 -0
data/lib/dca/helpers/logger.rb +50 -0
data/lib/dca/jobs.rb +3 -0
data/lib/dca/jobs/analyzer_job.rb +119 -0
data/lib/dca/jobs/job.rb +62 -0
data/lib/dca/models.rb +5 -0
data/lib/dca/models/base_model.rb +73 -0
data/lib/dca/models/binder.rb +68 -0
data/lib/dca/models/binder_helper.rb +48 -0
data/lib/dca/models/nokogiri_binder.rb +43 -0
data/lib/dca/models/position.rb +15 -0
data/lib/dca/net.rb +1 -0
data/lib/dca/net/browser_helper.rb +20 -0
data/lib/dca/notifier.rb +2 -0
data/lib/dca/notifier/notifier.rb +11 -0
data/lib/dca/notifier/redis/models/analyze_notify.rb +12 -0
data/lib/dca/notifier/redis/models/failure_notify.rb +8 -0
data/lib/dca/notifier/redis/models/fetch_notify.rb +15 -0
data/lib/dca/notifier/redis/models/session.rb +52 -0
data/lib/dca/notifier/redis/notifier.rb +25 -0
data/lib/dca/notifier/redis_notifier.rb +9 -0
data/lib/dca/storage.rb +3 -0
data/lib/dca/storage/elasticsearch_storage.rb +80 -0
data/lib/dca/storage/mongo_storage.rb +51 -0
data/lib/dca/storage/storage.rb +55 -0
data/spec/analyzer_spec.rb +64 -0
data/spec/area_task_spec.rb +45 -0
data/spec/base_model_spec.rb +34 -0
data/spec/binder_spec.rb +69 -0
data/spec/config.yml +18 -0
data/spec/elasticsearch_storage_spec.rb +28 -0
data/spec/fixtures/page.html +12 -0
data/spec/fixtures/positions.yml +13 -0
data/spec/fixtures/positions_with_error.yml +14 -0
data/spec/fixtures/states.yml +3 -0
data/spec/job_spec.rb +31 -0
data/spec/mock/analyzer_job.rb +30 -0
data/spec/mock/file_storage.rb +28 -0
data/spec/mock/notify_object.rb +13 -0
data/spec/mock/page.rb +13 -0
data/spec/mock/position.rb +40 -0
data/spec/mock/web_notifier.rb +30 -0
data/spec/mongo_storage_spec.rb +20 -0
data/spec/redis_notifier_spec.rb +98 -0
data/spec/spec_helper.rb +27 -0
data/spec/support/storage_examples.rb +103 -0
metadata +408 -0

@@ -0,0 +1,18 @@
+test:
+  elascticseach_db:
+    driver: ElasticSearch
+    host: localhost
+    port: 9200
+    index: test
+  mongo_db:
+    driver: Mongo
+    host: localhost
+    port: 27017
+    collection: test
+  db:
+    driver: Mock::File
+  notifier:
+    driver: Mock::Web
+  logger: false

data/spec/elasticsearch_storage_spec.rb ADDED

@@ -0,0 +1,28 @@
+require File.expand_path('../spec_helper', __FILE__)
+require File.expand_path('../mock/position', __FILE__)
+include DCA::Mock
+describe 'ElasticSearch storage' do
+  let(:connection) { @connection ||= DCA::ElasticSearchStorage.establish_connection APP_CONFIG[:elascticseach_db] }
+  let(:position) { ElasticSearchPosition.new :base_id => '0', :checksum => '0'}
+  let(:storage) { @storage ||= DCA::ElasticSearchStorage.new connection, position.class, :index => 'test' }
+  before :all do
+    connection
+    storage.index do
+      create
+      store :type => 'position', :base_id => '1', :checksum => '1'
+      refresh
+    end
+  end
+  after :all do
+    storage.index do
+      delete
+      refresh
+    end
+  end
+  it_behaves_like 'storage'
+end

data/spec/fixtures/page.html ADDED

@@ -0,0 +1,12 @@
+<html>
+<head>
+    <title>Page</title>
+</head>
+<body>
+<ul>
+    <li><a href="/positions/1">Position 1</a><span class="description">Description 1</span><span class="date">12.10.2012</span></li>
+    <li><a href="/positions/2">Position 2</a><span class="description">Description 2</span><span class="date">13.10.2012</span></li>
+    <li><a href="/positions/3">Position 3</a><span class="description">Description 3</span><span class="date">14.10.2012</span></li>
+</ul>
+</body>
+</html>

data/spec/fixtures/positions.yml ADDED

@@ -0,0 +1,13 @@
+---
+- id: 1
+  checksum: '1'
+  failed: false
+  base_id: 1
+- id: 2
+  checksum: '2'
+  failed: true
+  base_id: 2
+- id: 3
+  checksum: '2'
+  failed: false
+  base_id: 3

data/spec/fixtures/positions_with_error.yml ADDED

@@ -0,0 +1,14 @@
+---
+- id: 1
+  checksum: '1'
+  failed: false
+  base_id: 1
+- id: 2
+  checksum: '2'
+  failed: true
+  raise: true
+  base_id: 2
+- id: 3
+  checksum: '2'
+  failed: false
+  base_id: 3

data/spec/fixtures/states.yml ADDED

@@ -0,0 +1,3 @@
+1: create
+2: create
+3: unmodified

data/spec/job_spec.rb ADDED

@@ -0,0 +1,31 @@
+require 'timeout'
+require File.expand_path('../spec_helper', __FILE__)
+module TestModule
+  class TestJob < DCA::Jobs::Job
+  end
+  class LoopJob < DCA::Jobs::Job
+    def perform
+      loop do
+        sleep 1
+        break if shutdown?
+      end
+    end
+  end
+end
+describe 'Job' do
+  it 'should get queue name from module name' do
+    TestModule::TestJob.queue.should == 'TestModule'
+  end
+  it 'should shutdown when QUIT signal is happened' do
+    job_pid = Process.fork { TestModule::LoopJob.create }
+    sleep 1
+    Process.kill 'QUIT', job_pid
+    Timeout::timeout(1) {
+      Process.waitpid job_pid
+    }
+  end
+end

data/spec/mock/analyzer_job.rb ADDED

@@ -0,0 +1,30 @@
+module DCA
+  module Mock
+    class AnalyzerJob < DCA::Jobs::AnalyzerJob
+      def positions(&block)
+        positions = YAML.load_file(options[:fixture])
+        positions.each { |position| block.call Position.new position.symbolize_keys }
+      end
+      def fetch(position)
+        raise Exception if position.raise
+        position.failed ? nil : position
+      end
+    end
+  end
+  module Areas
+    module Mock
+      class AnalyzerJob < DCA::Jobs::AnalyzerJob
+        def perform
+          loop do
+            break if shutdown?
+          end
+        end
+      end
+    end
+  end
+end

data/spec/mock/file_storage.rb ADDED

@@ -0,0 +1,28 @@
+module DCA
+  module Mock
+    class FileStorage
+      attr_reader :collection
+      def initialize(connection, context, options = {})
+        @collection = DCA.project_name
+        @states = YAML.load_file('./spec/fixtures/states.yml')
+      end
+      def self.establish_connection(config)
+      end
+      def state(position)
+        @states[position.id].to_sym
+      end
+      def refresh(position, state)
+      end
+      def context object
+        self.clone
+      end
+    end
+  end
+end

data/spec/mock/notify_object.rb ADDED

@@ -0,0 +1,13 @@
+module DCA
+  module Mock
+    class NotifyObject
+      def session
+        'test_session'
+      end
+      def self.queue
+        'test_queue'
+      end
+    end
+  end
+end

data/spec/mock/page.rb ADDED

@@ -0,0 +1,13 @@
+module DCA
+  module Mock
+    class Page < DCA::Models::BaseModel
+      has_many :positions, ExtPosition, :selector => 'li'
+    end
+    class PageExt < DCA::Models::BaseModel
+      attr_accessor :category
+      has_many :positions, :selector => 'li', :polymorphic => :category, :append => true
+    end
+  end
+end

data/spec/mock/position.rb ADDED

@@ -0,0 +1,40 @@
+module DCA
+  module Mock
+    class MongoSearchPosition < DCA::Models::Position
+      establish_connection :mongo_db
+    end
+    class ElasticSearchPosition < DCA::Models::Position
+      establish_connection :elascticseach_db
+    end
+    class Position < DCA::Models::Position
+      attr_accessor :raise, :failed, :title
+      has_one :base_id, :integer, :selector => 'a', :attribute => :href, :regex => /(\d+)$/
+      has_one :title,   :string,  :selector => 'a'
+    end
+    class ExtPosition < Position
+      has_one :description, :string, :selector => 'span.description'
+      has_one :date,        :datetime, :selector => 'span.date'
+    end
+    class FullPosition < Position
+      has_one :base_id, :string, :selector => 'a'
+    end
+    class ChildPosition < DCA::Models::Position
+      attr_reader :name, :test
+    end
+    class RootPosition < DCA::Models::Position
+      has_one :one_child, :child_position
+      has_many :child_position
+    end
+    class PositionWithoutState < DCA::Models::BaseModel
+      attr_reader :name
+    end
+  end
+end

data/spec/mock/web_notifier.rb ADDED

@@ -0,0 +1,30 @@
+module DCA
+  module Mock
+    class WebNotifier
+      def initialize config
+      end
+      def self.queue
+        @queue ||= {}
+      end
+      def self.clean
+        @queue = {}
+      end
+      def push(object, event, options)
+        if event == :fetch && options[:result] == false
+          failed_queue = Mock::WebNotifier.queue[:failed] ||= {}
+          failed_queue[options[:state]] ||= 0
+          failed_queue[options[:state]] += 1
+        end
+        if [:analyze, :fetch].include? event
+          queue = Mock::WebNotifier.queue[event] ||= {}
+          queue[options[:state]] ||= 0
+          queue[options[:state]] += 1
+        end
+      end
+    end
+  end
+end

data/spec/mongo_storage_spec.rb ADDED

@@ -0,0 +1,20 @@
+require File.expand_path('../spec_helper', __FILE__)
+require File.expand_path('../mock/position', __FILE__)
+include DCA::Mock
+describe DCA::MongoStorage do
+  let(:connection) { @connection ||= DCA::MongoStorage.establish_connection APP_CONFIG[:mongo_db] }
+  let(:position) { MongoSearchPosition.new :base_id => '0', :checksum => '0'}
+  let(:storage) { @storage ||= DCA::MongoStorage.new connection, position.class, :collection => 'test' }
+  before :all do
+    connection
+  end
+  after :all do
+    connection.drop_database storage.database.name
+  end
+  it_behaves_like 'storage'
+end

data/spec/redis_notifier_spec.rb ADDED

@@ -0,0 +1,98 @@
+require File.expand_path('../spec_helper', __FILE__)
+require File.expand_path('../mock/notify_object', __FILE__)
+require File.expand_path('../mock/position', __FILE__)
+require File.expand_path('../mock/file_storage', __FILE__)
+require File.expand_path('../mock/analyzer_job', __FILE__)
+include DCA
+describe 'Redis Notifier' do
+  it 'should connect to redis' do
+    DCA::Notifier.create :driver => 'Redis', :host => 'localhost', :port => '6379'
+    Ohm.redis.info
+  end
+  describe 'Instance' do
+    before :all do
+      DCA::Notifier.create :driver => 'Redis', :host => 'localhost', :port => '6379'
+    end
+    before :each do
+      DCA::Redis::Session.all.each { |session| session.delete }
+    end
+    it 'should push analyze notify' do
+      notify_object = Mock::NotifyObject.new
+      DCA::Notifier.push notify_object, :analyze, :state => :create
+      session = DCA::Redis::Session.find(:project => 'DCA', :area => 'test_queue', :uid => 'test_session').first
+      session.analyze_state(:create).count.should equal 1
+      DCA::Notifier.push notify_object, :analyze, :state => :create
+      session.analyze_state(:create).count.should equal 2
+    end
+    it 'should push analyze notify with different state' do
+      notify_object = Mock::NotifyObject.new
+      DCA::Notifier.push notify_object, :analyze, :state => :create
+      DCA::Notifier.push notify_object, :analyze, :state => :update
+      DCA::Notifier.push notify_object, :analyze, :state => :remove
+      DCA::Notifier.push notify_object, :analyze, :state => :unmodified
+      session = DCA::Redis::Session.find(:project => 'DCA', :area => 'test_queue', :uid => 'test_session').first
+      session.analyze_state(:create).count.should equal 1
+      session.analyze_state(:update).count.should equal 1
+      session.analyze_state(:remove).count.should equal 1
+      session.analyze_state(:unmodified).count.should equal 1
+    end
+    it 'should push fetch notify' do
+      notify_object = Mock::NotifyObject.new
+      DCA::Notifier.push notify_object, :fetch, :state => :create, :result => true
+      session = DCA::Redis::Session.find(:project => 'DCA', :area => 'test_queue', :uid => 'test_session').first
+      session.fetch_state(:create).success.should equal 1
+      DCA::Notifier.push notify_object, :fetch, :state => :create, :result => true
+      session.fetch_state(:create).success.should equal 2
+      DCA::Notifier.push notify_object, :fetch, :state => :create, :result => false
+      session.fetch_state(:create).failure.should equal 1
+      DCA::Notifier.push notify_object, :fetch, :state => :create, :result => false
+      session.fetch_state(:create).failure.should equal 2
+    end
+    it 'should push analyze notify with different state' do
+      notify_object = Mock::NotifyObject.new
+      DCA::Notifier.push notify_object, :fetch, :state => :create, :result => true
+      DCA::Notifier.push notify_object, :fetch, :state => :update, :result => true
+      session = DCA::Redis::Session.find(:project => 'DCA', :area => 'test_queue', :uid => 'test_session').first
+      session.fetch_state(:create).success.should equal 1
+      session.fetch_state(:update).success.should equal 1
+    end
+    it 'should notify failure' do
+      notify_object = Mock::NotifyObject.new
+      begin
+        raise "Test exception"
+      rescue Exception => exception
+        DCA::Notifier.push notify_object, :failure, :exception => exception
+      end
+      session = DCA::Redis::Session.find(:project => 'DCA', :area => 'test_queue', :uid => 'test_session').first
+      session.failures.count.should equal 1
+    end
+    it 'should work with analyze job' do
+      Mock::AnalyzerJob.create :fixture => './spec/fixtures/positions.yml'
+      session = DCA::Redis::Session.all.first
+      session.analyze_state(:create).count.should equal 2
+      session.analyze_state(:unmodified).count.should equal 1
+      session.fetch_state(:create).success.should equal 1
+      session.fetch_state(:create).failure.should equal 1
+    end
+  end
+end

data/spec/spec_helper.rb ADDED

@@ -0,0 +1,27 @@
+require 'simplecov'
+SimpleCov.start
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'yaml'
+require 'rake'
+require 'rspec'
+require 'hashr'
+SYS_ENV = 'test'
+APP_CONFIG = YAML.load_file('./spec/config.yml')[SYS_ENV].deep_symbolize_keys
+require './lib/dca'
+# Set resque call perform method inline, without putting into redis queue
+Resque.inline = true
+# Requires supporting files with custom matchers and macros, etc,
+# in ./support/ and its subdirectories.
+Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
+RSpec.configure do |config|
+end

data/spec/support/storage_examples.rb ADDED

@@ -0,0 +1,103 @@
+shared_examples_for 'storage' do
+  context '.establish_connection' do
+    it 'should connect to storage' do
+      storage.should_not be_nil
+    end
+  end
+  describe '#state' do
+    context 'when new position' do
+      subject { position.state }
+      it { should equal :create}
+    end
+    context 'when modify position' do
+      before do
+        position.save
+        position.checksum = 1
+      end
+      after { position.destroy }
+      subject { position.state }
+      it { should equal :update}
+    end
+    context 'when exist position' do
+      before { position.save }
+      after { position.destroy }
+      subject { position.state }
+      it { should equal :unmodified}
+    end
+    context 'when position without state' do
+      let(:position) { PositionWithoutState.new name: 'test'}
+      subject { position.state }
+      it { should equal :create}
+    end
+  end
+  describe '#refresh' do
+    def refresh state
+      storage.should_receive(state).with(position)
+      storage.refresh(position, state)
+    end
+    context 'when new position' do
+      it 'then create it' do
+        refresh :create
+      end
+    end
+    context 'when modify position' do
+      it 'then update it' do
+        refresh :update
+      end
+    end
+    context 'when old position' do
+      it 'then delete it' do
+        refresh :remove
+      end
+    end
+  end
+  describe '#create' do
+    before { position.save }
+    after { position.destroy }
+    it 'sould create position' do
+      storage.find(position).should_not be_nil
+    end
+    it 'set position id' do
+      position.id.should_not be_nil
+    end
+  end
+  describe '#update' do
+    before do
+      position.save
+      position.checksum = '1'
+      position.save
+    end
+    after { position.destroy }
+    it 'should update position' do
+      storage.find(position)['checksum'].should eql '1'
+    end
+  end
+  describe '#remove' do
+    before do
+      position.save
+      position.destroy
+    end
+    after { position.destroy }
+    it 'should remove position' do
+      storage.find(position).should be_nil
+    end
+  end
+end