RubyGems - dca - Versions diffs - 0.1.0 - Mend

dca 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (70) hide show

data/.document +5 -0
data/.rspec +2 -0
data/Gemfile +48 -0
data/Gemfile.lock +126 -0
data/Guardfile +8 -0
data/LICENSE.txt +20 -0
data/README.rdoc +19 -0
data/Rakefile +48 -0
data/VERSION +1 -0
data/bin/dca +5 -0
data/dca.gemspec +160 -0
data/lib/dca.rb +64 -0
data/lib/dca/cli.rb +32 -0
data/lib/dca/commands/area.rb +133 -0
data/lib/dca/commands/templates/area/analyzer.rb.erb +34 -0
data/lib/dca/commands/templates/area/area.rb.erb +2 -0
data/lib/dca/commands/templates/area/models.rb.erb +2 -0
data/lib/dca/commands/templates/area/page.rb.erb +17 -0
data/lib/dca/commands/templates/area/position.rb.erb +8 -0
data/lib/dca/commands/templates/config.yml.erb +38 -0
data/lib/dca/commands/templates/spec/analyzer_spec.rb.erb +15 -0
data/lib/dca/commands/templates/spec/spec_helper.rb.erb +2 -0
data/lib/dca/config.rb +20 -0
data/lib/dca/helpers.rb +2 -0
data/lib/dca/helpers/logger.rb +50 -0
data/lib/dca/jobs.rb +3 -0
data/lib/dca/jobs/analyzer_job.rb +119 -0
data/lib/dca/jobs/job.rb +62 -0
data/lib/dca/models.rb +5 -0
data/lib/dca/models/base_model.rb +73 -0
data/lib/dca/models/binder.rb +68 -0
data/lib/dca/models/binder_helper.rb +48 -0
data/lib/dca/models/nokogiri_binder.rb +43 -0
data/lib/dca/models/position.rb +15 -0
data/lib/dca/net.rb +1 -0
data/lib/dca/net/browser_helper.rb +20 -0
data/lib/dca/notifier.rb +2 -0
data/lib/dca/notifier/notifier.rb +11 -0
data/lib/dca/notifier/redis/models/analyze_notify.rb +12 -0
data/lib/dca/notifier/redis/models/failure_notify.rb +8 -0
data/lib/dca/notifier/redis/models/fetch_notify.rb +15 -0
data/lib/dca/notifier/redis/models/session.rb +52 -0
data/lib/dca/notifier/redis/notifier.rb +25 -0
data/lib/dca/notifier/redis_notifier.rb +9 -0
data/lib/dca/storage.rb +3 -0
data/lib/dca/storage/elasticsearch_storage.rb +80 -0
data/lib/dca/storage/mongo_storage.rb +51 -0
data/lib/dca/storage/storage.rb +55 -0
data/spec/analyzer_spec.rb +64 -0
data/spec/area_task_spec.rb +45 -0
data/spec/base_model_spec.rb +34 -0
data/spec/binder_spec.rb +69 -0
data/spec/config.yml +18 -0
data/spec/elasticsearch_storage_spec.rb +28 -0
data/spec/fixtures/page.html +12 -0
data/spec/fixtures/positions.yml +13 -0
data/spec/fixtures/positions_with_error.yml +14 -0
data/spec/fixtures/states.yml +3 -0
data/spec/job_spec.rb +31 -0
data/spec/mock/analyzer_job.rb +30 -0
data/spec/mock/file_storage.rb +28 -0
data/spec/mock/notify_object.rb +13 -0
data/spec/mock/page.rb +13 -0
data/spec/mock/position.rb +40 -0
data/spec/mock/web_notifier.rb +30 -0
data/spec/mongo_storage_spec.rb +20 -0
data/spec/redis_notifier_spec.rb +98 -0
data/spec/spec_helper.rb +27 -0
data/spec/support/storage_examples.rb +103 -0
metadata +408 -0

data/spec/config.yml ADDED

@@ -0,0 +1,18 @@
+test:
+  elascticseach_db:
+    driver: ElasticSearch
+    host: localhost
+    port: 9200
+    index: test
+  mongo_db:
+    driver: Mongo
+    host: localhost
+    port: 27017
+    collection: test
+  db:
+    driver: Mock::File
+  notifier:
+    driver: Mock::Web
+  logger: false

data/spec/elasticsearch_storage_spec.rb ADDED

@@ -0,0 +1,28 @@
+require File.expand_path('../spec_helper', __FILE__)
+require File.expand_path('../mock/position', __FILE__)
+include DCA::Mock
+describe 'ElasticSearch storage' do
+  let(:connection) { @connection ||= DCA::ElasticSearchStorage.establish_connection APP_CONFIG[:elascticseach_db] }
+  let(:position) { ElasticSearchPosition.new :base_id => '0', :checksum => '0'}
+  let(:storage) { @storage ||= DCA::ElasticSearchStorage.new connection, position.class, :index => 'test' }
+  before :all do
+    connection
+    storage.index do
+      create
+      store :type => 'position', :base_id => '1', :checksum => '1'
+      refresh
+    end
+  end
+  after :all do
+    storage.index do
+      delete
+      refresh
+    end
+  end
+  it_behaves_like 'storage'
+end

data/spec/fixtures/page.html ADDED

@@ -0,0 +1,12 @@
+<html>
+<head>
+    <title>Page</title>
+</head>
+<body>
+<ul>
+    <li><a href="/positions/1">Position 1</a><span class="description">Description 1</span><span class="date">12.10.2012</span></li>
+    <li><a href="/positions/2">Position 2</a><span class="description">Description 2</span><span class="date">13.10.2012</span></li>
+    <li><a href="/positions/3">Position 3</a><span class="description">Description 3</span><span class="date">14.10.2012</span></li>
+</ul>
+</body>
+</html>

data/spec/fixtures/positions.yml ADDED

@@ -0,0 +1,13 @@
+---
+- id: 1
+  checksum: '1'
+  failed: false
+  base_id: 1
+- id: 2
+  checksum: '2'
+  failed: true
+  base_id: 2
+- id: 3
+  checksum: '2'
+  failed: false
+  base_id: 3

data/spec/fixtures/positions_with_error.yml ADDED

@@ -0,0 +1,14 @@
+---
+- id: 1
+  checksum: '1'
+  failed: false
+  base_id: 1
+- id: 2
+  checksum: '2'
+  failed: true
+  raise: true
+  base_id: 2
+- id: 3
+  checksum: '2'
+  failed: false
+  base_id: 3

data/spec/fixtures/states.yml ADDED

@@ -0,0 +1,3 @@
+1: create
+2: create
+3: unmodified

data/spec/job_spec.rb ADDED

@@ -0,0 +1,31 @@
+require 'timeout'
+require File.expand_path('../spec_helper', __FILE__)
+module TestModule
+  class TestJob < DCA::Jobs::Job
+  end
+  class LoopJob < DCA::Jobs::Job
+    def perform
+      loop do
+        sleep 1
+        break if shutdown?
+      end
+    end
+  end
+end
+describe 'Job' do
+  it 'should get queue name from module name' do
+    TestModule::TestJob.queue.should == 'TestModule'
+  end
+  it 'should shutdown when QUIT signal is happened' do
+    job_pid = Process.fork { TestModule::LoopJob.create }
+    sleep 1
+    Process.kill 'QUIT', job_pid
+    Timeout::timeout(1) {
+      Process.waitpid job_pid
+    }
+  end
+end

data/spec/mock/analyzer_job.rb ADDED

@@ -0,0 +1,30 @@
+module DCA
+  module Mock
+    class AnalyzerJob < DCA::Jobs::AnalyzerJob
+      def positions(&block)
+        positions = YAML.load_file(options[:fixture])
+        positions.each { |position| block.call Position.new position.symbolize_keys }
+      end
+      def fetch(position)
+        raise Exception if position.raise
+        position.failed ? nil : position
+      end
+    end
+  end
+  module Areas
+    module Mock
+      class AnalyzerJob < DCA::Jobs::AnalyzerJob
+        def perform
+          loop do
+            break if shutdown?
+          end
+        end
+      end
+    end
+  end
+end

data/spec/mock/file_storage.rb ADDED

@@ -0,0 +1,28 @@
+module DCA
+  module Mock
+    class FileStorage
+      attr_reader :collection
+      def initialize(connection, context, options = {})
+        @collection = DCA.project_name
+        @states = YAML.load_file('./spec/fixtures/states.yml')
+      end
+      def self.establish_connection(config)
+      end
+      def state(position)
+        @states[position.id].to_sym
+      end
+      def refresh(position, state)
+      end
+      def context object
+        self.clone
+      end
+    end
+  end
+end

data/spec/mock/notify_object.rb ADDED

@@ -0,0 +1,13 @@
+module DCA
+  module Mock
+    class NotifyObject
+      def session
+        'test_session'
+      end
+      def self.queue
+        'test_queue'
+      end
+    end
+  end
+end

data/spec/mock/page.rb ADDED

@@ -0,0 +1,13 @@
+module DCA
+  module Mock
+    class Page < DCA::Models::BaseModel
+      has_many :positions, ExtPosition, :selector => 'li'
+    end
+    class PageExt < DCA::Models::BaseModel
+      attr_accessor :category
+      has_many :positions, :selector => 'li', :polymorphic => :category, :append => true
+    end
+  end
+end

data/spec/mock/position.rb ADDED

@@ -0,0 +1,40 @@
+module DCA
+  module Mock
+    class MongoSearchPosition < DCA::Models::Position
+      establish_connection :mongo_db
+    end
+    class ElasticSearchPosition < DCA::Models::Position
+      establish_connection :elascticseach_db
+    end
+    class Position < DCA::Models::Position
+      attr_accessor :raise, :failed, :title
+      has_one :base_id, :integer, :selector => 'a', :attribute => :href, :regex => /(\d+)$/
+      has_one :title,   :string,  :selector => 'a'
+    end
+    class ExtPosition < Position
+      has_one :description, :string, :selector => 'span.description'
+      has_one :date,        :datetime, :selector => 'span.date'
+    end
+    class FullPosition < Position
+      has_one :base_id, :string, :selector => 'a'
+    end
+    class ChildPosition < DCA::Models::Position
+      attr_reader :name, :test
+    end
+    class RootPosition < DCA::Models::Position
+      has_one :one_child, :child_position
+      has_many :child_position
+    end
+    class PositionWithoutState < DCA::Models::BaseModel
+      attr_reader :name
+    end
+  end
+end

data/spec/mock/web_notifier.rb ADDED

@@ -0,0 +1,30 @@
+module DCA
+  module Mock
+    class WebNotifier
+      def initialize config
+      end
+      def self.queue
+        @queue ||= {}
+      end
+      def self.clean
+        @queue = {}
+      end
+      def push(object, event, options)
+        if event == :fetch && options[:result] == false
+          failed_queue = Mock::WebNotifier.queue[:failed] ||= {}
+          failed_queue[options[:state]] ||= 0
+          failed_queue[options[:state]] += 1
+        end
+        if [:analyze, :fetch].include? event
+          queue = Mock::WebNotifier.queue[event] ||= {}
+          queue[options[:state]] ||= 0
+          queue[options[:state]] += 1
+        end
+      end
+    end
+  end
+end

data/spec/mongo_storage_spec.rb ADDED

@@ -0,0 +1,20 @@
+require File.expand_path('../spec_helper', __FILE__)
+require File.expand_path('../mock/position', __FILE__)
+include DCA::Mock
+describe DCA::MongoStorage do
+  let(:connection) { @connection ||= DCA::MongoStorage.establish_connection APP_CONFIG[:mongo_db] }
+  let(:position) { MongoSearchPosition.new :base_id => '0', :checksum => '0'}
+  let(:storage) { @storage ||= DCA::MongoStorage.new connection, position.class, :collection => 'test' }
+  before :all do
+    connection
+  end
+  after :all do
+    connection.drop_database storage.database.name
+  end
+  it_behaves_like 'storage'
+end

data/spec/redis_notifier_spec.rb ADDED

@@ -0,0 +1,98 @@
+require File.expand_path('../spec_helper', __FILE__)
+require File.expand_path('../mock/notify_object', __FILE__)
+require File.expand_path('../mock/position', __FILE__)
+require File.expand_path('../mock/file_storage', __FILE__)
+require File.expand_path('../mock/analyzer_job', __FILE__)
+include DCA
+describe 'Redis Notifier' do
+  it 'should connect to redis' do
+    DCA::Notifier.create :driver => 'Redis', :host => 'localhost', :port => '6379'
+    Ohm.redis.info
+  end
+  describe 'Instance' do
+    before :all do
+      DCA::Notifier.create :driver => 'Redis', :host => 'localhost', :port => '6379'
+    end
+    before :each do
+      DCA::Redis::Session.all.each { |session| session.delete }
+    end
+    it 'should push analyze notify' do
+      notify_object = Mock::NotifyObject.new
+      DCA::Notifier.push notify_object, :analyze, :state => :create
+      session = DCA::Redis::Session.find(:project => 'DCA', :area => 'test_queue', :uid => 'test_session').first
+      session.analyze_state(:create).count.should equal 1
+      DCA::Notifier.push notify_object, :analyze, :state => :create
+      session.analyze_state(:create).count.should equal 2
+    end
+    it 'should push analyze notify with different state' do
+      notify_object = Mock::NotifyObject.new
+      DCA::Notifier.push notify_object, :analyze, :state => :create
+      DCA::Notifier.push notify_object, :analyze, :state => :update
+      DCA::Notifier.push notify_object, :analyze, :state => :remove
+      DCA::Notifier.push notify_object, :analyze, :state => :unmodified
+      session = DCA::Redis::Session.find(:project => 'DCA', :area => 'test_queue', :uid => 'test_session').first
+      session.analyze_state(:create).count.should equal 1
+      session.analyze_state(:update).count.should equal 1
+      session.analyze_state(:remove).count.should equal 1
+      session.analyze_state(:unmodified).count.should equal 1
+    end
+    it 'should push fetch notify' do
+      notify_object = Mock::NotifyObject.new
+      DCA::Notifier.push notify_object, :fetch, :state => :create, :result => true
+      session = DCA::Redis::Session.find(:project => 'DCA', :area => 'test_queue', :uid => 'test_session').first
+      session.fetch_state(:create).success.should equal 1
+      DCA::Notifier.push notify_object, :fetch, :state => :create, :result => true
+      session.fetch_state(:create).success.should equal 2
+      DCA::Notifier.push notify_object, :fetch, :state => :create, :result => false
+      session.fetch_state(:create).failure.should equal 1
+      DCA::Notifier.push notify_object, :fetch, :state => :create, :result => false
+      session.fetch_state(:create).failure.should equal 2
+    end
+    it 'should push analyze notify with different state' do
+      notify_object = Mock::NotifyObject.new
+      DCA::Notifier.push notify_object, :fetch, :state => :create, :result => true
+      DCA::Notifier.push notify_object, :fetch, :state => :update, :result => true
+      session = DCA::Redis::Session.find(:project => 'DCA', :area => 'test_queue', :uid => 'test_session').first
+      session.fetch_state(:create).success.should equal 1
+      session.fetch_state(:update).success.should equal 1
+    end
+    it 'should notify failure' do
+      notify_object = Mock::NotifyObject.new
+      begin
+        raise "Test exception"
+      rescue Exception => exception
+        DCA::Notifier.push notify_object, :failure, :exception => exception
+      end
+      session = DCA::Redis::Session.find(:project => 'DCA', :area => 'test_queue', :uid => 'test_session').first
+      session.failures.count.should equal 1
+    end
+    it 'should work with analyze job' do
+      Mock::AnalyzerJob.create :fixture => './spec/fixtures/positions.yml'
+      session = DCA::Redis::Session.all.first
+      session.analyze_state(:create).count.should equal 2
+      session.analyze_state(:unmodified).count.should equal 1
+      session.fetch_state(:create).success.should equal 1
+      session.fetch_state(:create).failure.should equal 1
+    end
+  end
+end

data/spec/spec_helper.rb ADDED

@@ -0,0 +1,27 @@
+require 'simplecov'
+SimpleCov.start
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'yaml'
+require 'rake'
+require 'rspec'
+require 'hashr'
+SYS_ENV = 'test'
+APP_CONFIG = YAML.load_file('./spec/config.yml')[SYS_ENV].deep_symbolize_keys
+require './lib/dca'
+# Set resque call perform method inline, without putting into redis queue
+Resque.inline = true
+# Requires supporting files with custom matchers and macros, etc,
+# in ./support/ and its subdirectories.
+Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
+RSpec.configure do |config|
+end

data/spec/support/storage_examples.rb ADDED

@@ -0,0 +1,103 @@
+shared_examples_for 'storage' do
+  context '.establish_connection' do
+    it 'should connect to storage' do
+      storage.should_not be_nil
+    end
+  end
+  describe '#state' do
+    context 'when new position' do
+      subject { position.state }
+      it { should equal :create}
+    end
+    context 'when modify position' do
+      before do
+        position.save
+        position.checksum = 1
+      end
+      after { position.destroy }
+      subject { position.state }
+      it { should equal :update}
+    end
+    context 'when exist position' do
+      before { position.save }
+      after { position.destroy }
+      subject { position.state }
+      it { should equal :unmodified}
+    end
+    context 'when position without state' do
+      let(:position) { PositionWithoutState.new name: 'test'}
+      subject { position.state }
+      it { should equal :create}
+    end
+  end
+  describe '#refresh' do
+    def refresh state
+      storage.should_receive(state).with(position)
+      storage.refresh(position, state)
+    end
+    context 'when new position' do
+      it 'then create it' do
+        refresh :create
+      end
+    end
+    context 'when modify position' do
+      it 'then update it' do
+        refresh :update
+      end
+    end
+    context 'when old position' do
+      it 'then delete it' do
+        refresh :remove
+      end
+    end
+  end
+  describe '#create' do
+    before { position.save }
+    after { position.destroy }
+    it 'sould create position' do
+      storage.find(position).should_not be_nil
+    end
+    it 'set position id' do
+      position.id.should_not be_nil
+    end
+  end
+  describe '#update' do
+    before do
+      position.save
+      position.checksum = '1'
+      position.save
+    end
+    after { position.destroy }
+    it 'should update position' do
+      storage.find(position)['checksum'].should eql '1'
+    end
+  end
+  describe '#remove' do
+    before do
+      position.save
+      position.destroy
+    end
+    after { position.destroy }
+    it 'should remove position' do
+      storage.find(position).should be_nil
+    end
+  end
+end