dca 0.1.0

data/lib/dca.rb

@@ -0,0 +1,64 @@
+require 'yaml'
+require 'active_model'
+require 'rake'
+require 'resque'
+require 'resque/tasks'
+require 'ohm'
+require 'uuid'
+require 'yajl/json_gem'
+require 'tire'
+require 'logger'
+require 'thor'
+require 'thor/group'
+require 'nokogiri'
+require 'mongo'
+
+module DCA
+  class ApplicationError < Exception; end
+
+  def self.root
+    Dir.pwd
+  end
+
+  def self.used?
+    File.basename(self.root).downcase != 'dca'
+  end
+
+  def self.project_name
+    return @project if @project.present?
+
+    gemspec = Dir[File.join self.root, '*.gemspec'].first
+    raise 'Generate gemspec file' if gemspec.blank?
+
+    gem = Gem::Specification.load gemspec
+    raise 'Set gem name in gemspec'  if gem.name.blank?
+
+    @project = gem.name.camelize
+    if @project.safe_constantize.nil?
+      @project = (Object.constants.detect { |const|
+        const.to_s.downcase == @project.downcase}).to_s
+      raise "Unknown project name" if @project.nil?
+    end
+
+    @project
+  end
+
+  def self.project_path
+    @project_path ||= File.join(DCA.root, 'lib', File.basename(DCA.root))
+  end
+
+  def self.project_file
+    @project_path ||= project_path + '.rb'
+  end
+end
+
+require File.expand_path('../dca/config', __FILE__)
+require File.expand_path('../dca/helpers', __FILE__)
+require File.expand_path('../dca/storage', __FILE__)
+require File.expand_path('../dca/jobs', __FILE__)
+require File.expand_path('../dca/net', __FILE__)
+require File.expand_path('../dca/notifier', __FILE__)
+require File.expand_path('../dca/models', __FILE__)
+require File.expand_path('../dca/cli', __FILE__)
+
+require DCA.project_path if DCA.used? && File.exist?(DCA.project_file)

data/lib/dca/cli.rb

@@ -0,0 +1,32 @@
+require File.expand_path('../commands/area', __FILE__)
+
+module DCA
+  class CLI < Thor
+    include Thor::Actions
+
+    def self.source_root
+      File.expand_path('../commands/templates', __FILE__)
+    end
+
+    desc 'config', 'Create default config file'
+    def config
+      empty_directory 'config'
+      template 'config.yml.erb', 'config/config.yml'
+    end
+
+    desc 'area SUBCOMMAND ...ARGS', 'Manage project areas'
+    subcommand 'area', Commands::Area
+
+    desc 'install', 'Install dca project'
+    def install
+      project = "#{DCA.project_name}::Project".constantize
+      project.install
+    end
+
+    desc 'uninstall', 'Uninstall dca  project'
+    def uninstall
+      project = "#{DCA.project_name}::Project".constantize
+      project.remove
+    end
+  end
+end

data/lib/dca/commands/area.rb

@@ -0,0 +1,133 @@
+module DCA
+  module Commands
+    class Area < Thor
+      include Thor::Actions
+
+      def self.source_root
+        File.expand_path('../templates', __FILE__)
+      end
+
+      desc 'create NAME TITLE DESCRIPTION URL', 'Generate and add new area to project'
+      long_desc <<-LONGDESC
+        Generate and add a new area named <name> with to project. Parameters <name>, <title>, <description> and <url>
+        are required
+      LONGDESC
+      def create name, title, description, url
+        @name = name
+        @class_name = name.camelize
+
+        template 'area/analyzer.rb.erb', "lib/areas/#{name.downcase}/analyzer.rb"
+        template 'area/position.rb.erb', "lib/areas/#{name.downcase}/models/position.rb"
+        template 'area/page.rb.erb', "lib/areas/#{name.downcase}/models/page.rb"
+        template 'area/models.rb.erb', "lib/areas/#{name.downcase}/models.rb"
+        template 'area/area.rb.erb', "lib/areas/#{name.downcase}.rb"
+
+        template 'spec/analyzer_spec.rb.erb', "spec/areas/#{name.downcase}/analyzer_spec.rb"
+        template 'spec/spec_helper.rb.erb', "spec/areas/#{name.downcase}/spec_helper.rb"
+
+        empty_directory 'config'
+        area_file = 'config/areas.yml'
+        areas = {}
+        areas = YAML.load_file(area_file) if File.exist? area_file
+        area_hash = {'title' => title, 'description' => description, 'url' => url}
+        if areas.has_key? name
+          if areas[name] == area_hash
+            shell.say_status :identical, area_file, :blue
+          else
+            areas[name] = area_hash
+            shell.say_status :conflict, area_file, :red
+            if shell.file_collision(area_file) { areas.to_yaml }
+              open(area_file, 'w:utf-8') { |file| file.write areas.to_yaml }
+              shell.say_status :force, area_file, :yellow
+            end
+          end
+        else
+          status = areas.empty? ? :create : :update
+          areas[name] = area_hash
+          open(area_file, 'w:utf-8') { |file| file.write areas.to_yaml }
+          shell.say_status status, area_file, :green
+        end
+      end
+
+      desc 'start NAME', 'Start area to analyze'
+      def start name
+        shell.say "Starting analyze area #{name}"
+        config = area_config name.to_sym
+
+        job_ops = {}
+        job_ops[:distributed] = true if config[:distributed]
+        job = "#{DCA.project_name}::Areas::#{name}::AnalyzerJob".constantize
+        job.create job_ops
+
+        background = config[:background].nil? ? true : config[:background]
+        run_worker name, config[:workers] || 1, background
+      end
+
+      desc 'stop NAME', 'Stop area to analyze'
+      method_option :force, :type => :boolean, :aliases => '-f', :desc => 'force stop area analyzing process'
+      def stop name
+        shell.say "Stopping analyze area #{name}"
+
+        pids = workers_pids name
+        unless pids.empty?
+          system("kill -s #{options[:force] ? 'TERM' : 'QUIT'} #{pids.join(' ')}")
+        end
+
+        wait_worker name
+
+        Resque.remove_queue name
+      end
+
+      private
+
+      def area_config area_name
+        config = {}
+        config = APP_CONFIG[:areas][area_name] if APP_CONFIG[:areas]
+        config
+      end
+
+      def run_worker(queue, count = 1, background = true)
+        puts "Starting #{count} worker(s) with QUEUE: #{queue}"
+        unless background
+          ENV['QUEUE'] = queue
+          ENV['VERBOSE'] = '1'
+          Rake::Task['resque:work'].invoke
+        else
+          log_dir = File.join DCA.root, 'log'
+          Dir.mkdir log_dir unless Dir.exist? log_dir
+          ops = { :pgroup => true }
+          if APP_CONFIG[:logger]
+            debug_file = [File.join(DCA.root, "log/#{queue.underscore}.debug"), 'a']
+            ops[:err] = debug_file
+            ops[:out] = debug_file
+          end
+          env_vars = {'QUEUE' => queue}
+
+          count.times {
+            ## Using Kernel.spawn and Process.detach because regular system() call would
+            ## cause the processes to quit when capistrano finishes
+            pid = spawn(env_vars, "rake resque:work", ops)
+            Process.detach(pid)
+          }
+        end
+      end
+
+      def workers_pids name
+        pids = Array.new
+        Resque.workers.each do |worker|
+          host, pid, queues = worker.id.split(':')
+          next unless host == worker.hostname
+
+          queues = queues.split(',')
+          next unless queues.include? name
+          pids.concat(worker.worker_pids)
+        end
+        pids.uniq
+      end
+
+      def wait_worker name
+        sleep 1 while workers_pids(name).count > 0
+      end
+    end
+  end
+end

data/lib/dca/commands/templates/area/analyzer.rb.erb

@@ -0,0 +1,34 @@
+module <%= DCA.project_name %>
+  module Areas
+    module <%= @class_name %>
+      # Base analyze jobs class for <%=@class_name%> area.
+      class AnalyzerJob < DCA::Jobs::AnalyzerJob
+        def change
+          false
+        end
+
+        def positions &block
+          page = Page.find options[:category]
+          until page.nil? do
+            logger.debug "Page number #{page.number}"
+
+            page.positions.each do |position|
+              begin
+                block.call position
+              rescue Exception => e
+                logger.exception e
+              end
+            end
+
+            break if shutdown?
+            break unless page.next
+          end
+        end
+
+        def fetch position
+          position
+        end
+      end
+    end
+  end
+end

data/lib/dca/commands/templates/area/area.rb.erb

@@ -0,0 +1,2 @@
+require File.expand_path('../<%= @name %>/models', __FILE__)
+require File.expand_path('../<%= @name %>/analyzer', __FILE__)

data/lib/dca/commands/templates/area/models.rb.erb

@@ -0,0 +1,2 @@
+require File.expand_path('../models/position', __FILE__)
+require File.expand_path('../models/page', __FILE__)

data/lib/dca/commands/templates/area/page.rb.erb

@@ -0,0 +1,17 @@
+module <%= DCA.project_name %>
+  module Areas
+    module <%= @class_name %>
+      class Page < DCA::Models::BaseModel
+        attr_reader :url, :number
+
+        # need to ser right selector
+        # has_many :positions, :selector => 'div'
+
+        def next
+          @number += 1
+          # get next page if it posible or return nil
+        end
+      end
+    end
+  end
+end

data/lib/dca/commands/templates/area/position.rb.erb

@@ -0,0 +1,8 @@
+module <%= DCA.project_name %>
+  module Areas
+    module <%= @class_name %>
+      class Position < DCA::Models::Position
+      end
+    end
+  end
+end

data/lib/dca/commands/templates/config.yml.erb

@@ -0,0 +1,38 @@
+production:
+  dca_db:
+    driver: ElasticSearch
+    host: localhost
+    port: 27017
+  redis:
+    host: localhost
+    port: 6379
+  notifier:
+    driver: Redis
+    host: localhost
+    port: 6379
+
+development:
+  dca_db:
+    driver: ElasticSearch
+    host: localhost
+    port: 27017
+  redis:
+    host: localhost
+    port: 6379
+  notifier:
+    driver: Redis
+    host: localhost
+    port: 6379
+
+test:
+  dca_db:
+    driver: ElasticSearch
+    host: localhost
+    port: 27017
+  redis:
+    host: localhost
+    port: 6379
+  notifier:
+    driver: Redis
+    host: localhost
+    port: 6379

data/lib/dca/commands/templates/spec/analyzer_spec.rb.erb

@@ -0,0 +1,15 @@
+require File.expand_path('../spec_helper', __FILE__)
+
+include <%= DCA.project_name%>::Areas::<%= @class_name %>
+
+describe '<%= @class_name%> analyzer' do
+  before :all do
+    <%= DCA.project_name %>::Project.remove
+    <%= DCA.project_name %>::Project.install
+    DCA::Notifier.create APP_CONFIG[:notifier]
+  end
+
+  after :all do
+    <%= DCA.project_name %>::Project.remove
+  end
+end

data/lib/dca/commands/templates/spec/spec_helper.rb.erb

@@ -0,0 +1,2 @@
+require File.expand_path('../../../spec_helper', __FILE__)
+require './lib/areas/<%= @name %>'

data/lib/dca/config.rb

@@ -0,0 +1,20 @@
+SYS_ENV = 'development' unless defined? SYS_ENV
+
+unless defined? APP_CONFIG
+  if File.exist? './config/config.yml'
+    APP_CONFIG = YAML.load_file('./config/config.yml')[SYS_ENV].deep_symbolize_keys
+  else
+    APP_CONFIG = {}
+    puts 'WARNING! Missing config file. Use rake system:config to create default config file.' if DCA.used?
+  end
+end
+
+unless defined? AREAS_CONFIG
+  if APP_CONFIG[:areas]
+    AREAS_CONFIG = APP_CONFIG[:areas]
+  else
+    AREAS_CONFIG = {}
+  end
+end
+
+

data/lib/dca/helpers.rb

@@ -0,0 +1,2 @@
+require File.expand_path('../helpers/logger', __FILE__)
+

data/lib/dca/helpers/logger.rb

@@ -0,0 +1,50 @@
+module DCA
+  module Helpers
+    class VerboseLogger < ::Logger
+
+      def initialize(logdev, shift_age = 0, shift_size = 1048576, verbose = false)
+        super logdev, shift_age, shift_size
+        @verbose_logdev = LogDevice.new(STDOUT, :shift_age => shift_age, :shift_size => shift_size) if verbose
+        @default_logdev = @logdev
+      end
+
+      def add(severity, message = nil, progname = nil, &block)
+        super severity, message, progname, &block
+
+        if @verbose_logdev
+          @logdev = @verbose_logdev
+          super severity, message, progname, &block
+          @logdev = @default_logdev
+        end
+      end
+
+      def exception(error, progname = nil, &block)
+        add(FATAL, "#{error.message}\n#{error.backtrace.join("\n")}", progname, &block)
+      end
+    end
+
+    module Logger
+      extend ActiveSupport::Concern
+
+      module ClassMethods
+        def logger_name name
+          define_method :logger do
+            @logger unless @logger.nil?
+
+            out = APP_CONFIG[:logger] ? File.join(DCA.root, 'log', "#{(self.class.send name).underscore}.log") : NIL
+            @logger ||= VerboseLogger.new out, 0, 1048576, APP_CONFIG[:verbose]
+          end
+        end
+
+        def logger= value
+          @logger = value
+        end
+      end
+
+      def logger
+        self.class.logger
+      end
+
+    end
+  end
+end

data/lib/dca/jobs.rb

@@ -0,0 +1,3 @@
+require File.expand_path('../jobs/job', __FILE__)
+require File.expand_path('../jobs/analyzer_job', __FILE__)
+

data/lib/dca/jobs/analyzer_job.rb

@@ -0,0 +1,119 @@
+module DCA
+  module Jobs
+    class AnalyzerJob < Job
+      def session
+        @session ||= options[:session] || UUID.generate(:compact)
+      end
+
+      def perform
+        return on_change if change
+
+        if options[:distributed] && options[:position]
+          analyze position options[:position]
+          return
+        end
+
+        index = 0
+        # get list of positions and add to cache
+        positions do |position|
+          if options[:distributed]
+            distribute position
+          else
+            analyze position
+          end
+
+          index += 1
+          break if options[:limit] == index || shutdown?
+        end
+      end
+
+      def change
+        false
+      end
+
+      def distribute position
+        self.class.create :distributed => true, :position => position.to_hash, session => self.session
+      end
+
+      # Return all positions or newly created or modified if possible. Some cases not possible to get newly created or
+      # modified positions. In this case cache will be used to identify only newly created or modified positions.
+      # Position must be a hash and should contain unique key :id and checksum for compare with cached positions and
+      # identify newly created or modified
+      def positions(&block)
+        raise NotImplementedError
+      end
+
+      # Return position model from hash
+      def position hash
+        Models::Position.new hash
+      end
+
+      # Fetch newly created or modified positions
+      def fetch position
+        raise NotImplementedError
+      end
+
+      def on_change
+        notify(:change)
+      end
+
+      def on_analyze(position, state)
+        logger.debug "[#{position.class}] Analyze position base_id:#{position.base_id} state:#{state}"
+        notify(:analyze, :position => position, :state => state)
+      end
+
+      def on_fetch(position, state, result)
+        if result
+          logger.debug "[#{position.class}] Fetch valid position id:#{position.id} base_id:#{position.base_id} state:#{state}"
+        else
+          logger.debug "[#{position.class}] Fetch invalid position base_id:#{position.base_id} state:#{state}"
+          logger.debug "  Validation errors:\n    #{position.errors.full_messages.join("\n    ")}"
+        end
+        notify(:fetch, :position => position, :state => state, :result => result )
+      end
+
+      def on_failure(error)
+        logger.exception error
+        notify(:failure, :exception => error)
+      end
+
+      def on_success
+        notify(:success)
+      end
+
+      protected
+
+      def notify(event, options={})
+        Notifier.push self, event, options
+      end
+
+      def analyze position
+        state = position.state
+        on_analyze position, state
+
+        unless state == :unmodified
+          new_position = fetch_safe! position
+
+          unless new_position
+            on_fetch position, state, false
+          else
+            position = new_position
+
+            valid = new_position.valid?
+            state = position.save if valid
+            on_fetch position, state, valid
+          end
+        end
+      rescue Exception => exception
+        on_failure exception
+      end
+
+      def fetch_safe!(position)
+        fetch position
+      rescue Exception => exception
+        on_failure exception
+        false
+      end
+    end
+  end
+end