dca 0.1.0

data/lib/dca/jobs/job.rb

@@ -0,0 +1,62 @@
+module DCA
+  module Jobs
+    class Job
+      include Helpers::Logger
+      logger_name :queue
+
+      attr_reader :options
+
+      def self.queue
+        self.to_s.split("::")[-2]
+      end
+
+      # Add a job to queue. Queue name is a class module name
+      def self.create(options={})
+        Resque.enqueue(self, options)
+      end
+
+      def self.perform(options={})
+        instance = new options.symbolize_keys!
+        instance.safe_perform!
+        instance
+      end
+
+      def initialize(options = {})
+        @options = options
+
+        trap('QUIT') { shutdown }
+      end
+
+      def safe_perform!
+        perform
+        on_success if respond_to?(:on_success)
+      rescue Exception => exception
+        if respond_to?(:on_failure)
+          on_failure(exception)
+        else
+          raise exception
+        end
+      ensure
+        destroy
+      end
+
+      def perform
+        raise NotImplementedError
+      end
+
+      def destroy
+
+      end
+
+      def shutdown?
+        @shutdown
+      end
+
+      private
+
+      def shutdown
+        @shutdown = true
+      end
+    end
+  end
+end

data/lib/dca/models.rb

@@ -0,0 +1,5 @@
+require File.expand_path('../models/binder', __FILE__)
+require File.expand_path('../models/binder_helper', __FILE__)
+require File.expand_path('../models/nokogiri_binder', __FILE__)
+require File.expand_path('../models/base_model', __FILE__)
+require File.expand_path('../models/position', __FILE__)

data/lib/dca/models/base_model.rb

@@ -0,0 +1,73 @@
+module DCA
+  module Models
+    class BaseModel
+      extend  ActiveModel::Naming
+      extend  ActiveModel::Translation
+      include ActiveModel::Validations
+      include ActiveModel::Conversion
+      include ActiveModel::Serialization
+      include Binder
+      include DCA::Storage
+
+      establish_connection
+
+      set_callback :validate, :before, :validate_associations
+
+      attr_accessor :id, :base_id, :created_at, :updated_at
+
+                    def initialize(params={})
+        params.each { |attr, value| self.instance_variable_set "@#{attr}", value } if params
+      end
+
+      def persisted?
+        true
+      end
+
+      def to_hash
+        include = []
+        self.class.associations(true).each { |field, options| include << field.to_s}
+        self.serializable_hash include: include
+      end
+
+      def attributes
+        return @attributes unless @attributes.nil?
+
+        @attributes = Hash[instance_variables.map { |var| [var.to_s.delete('@'), instance_variable_get(var)]}]
+        @attributes.delete 'errors'
+        @attributes.delete 'validation_context'
+
+        @attributes
+      end
+
+      def before_update
+        self.updated_at = Time.now.utc
+      end
+
+      def before_create
+        self.created_at = Time.now.utc
+      end
+
+      def validate_associations
+        self.class.associations.each do |field, options|
+          object = self.send(field)
+          next if object.nil?
+
+          if object.is_a? Array
+            object.each { |item| validate_child item, field }
+          else
+            validate_child object, field
+          end
+        end
+      end
+
+      private
+
+      def validate_child object, field
+        if object.respond_to?(:invalid?) && object.invalid?
+          self.errors.add field, object.errors.full_messages
+        end
+      end
+
+    end
+  end
+end

data/lib/dca/models/binder.rb

@@ -0,0 +1,68 @@
+module DCA
+  module Models
+    module Binder
+      extend ActiveSupport::Concern
+
+      COMPLEX_TYPE = [:integer, :float, :string, :symbol, :datetime]
+
+      module ClassMethods
+        def binder name = :nokogiri
+          @binder ||= "DCA::Models::#{name.to_s.camelize}Binder".constantize
+        end
+
+        def has_one field, *args
+          options  = args.extract_options!
+          type = args.first
+          add_association field, :one, type, options
+        end
+
+        def has_many field, *args
+          options  = args.extract_options!
+          type = args.first
+          add_association field, :many, type, options
+        end
+
+        def associations complex = false
+          @associations ||= {}
+
+          return associations.select { |field, options| !COMPLEX_TYPE.include?(options[:type])  } if complex
+
+          @associations
+        end
+
+        def inherited(child)
+          associations.each { |field, options| child.associations[field] = options}
+        end
+
+        private
+
+        def add_association field, association, type, options = {}
+          associations[field] = { :association => association, :field => field, :type => type, :options => options }
+          instance_eval do
+            attr_accessor field.to_sym unless instance_variable_defined? "@#{field}"
+          end
+        end
+      end
+
+      def bind content
+        self.class.associations.each do |field, options|
+          update field, self.class.binder.parse(self, content, options), options[:options][:append]
+        end
+        self
+      end
+
+      private
+
+      def update field, value, append = false
+        if append
+          new_value = self.instance_variable_get("@#{field.to_s}")
+          new_value  = new_value ? new_value + value : value
+          self.instance_variable_set "@#{field.to_s}", new_value
+        else
+          self.instance_variable_set "@#{field.to_s}", value
+        end
+      end
+    end
+  end
+end
+

data/lib/dca/models/binder_helper.rb

@@ -0,0 +1,48 @@
+module DCA
+  module Models
+    module BinderHelper
+      extend ActiveSupport::Concern
+
+      module ClassMethods
+        def convert value, type
+          case type
+            when :integer
+              value.to_s.gsub(/[^\d.,]/,'').gsub(/,/,'.').to_i
+            when :float
+              value.to_s.gsub(/[^\d.,]/,'').gsub(/,/,'.').to_f
+            when :string
+              value.to_s.strip
+            when :symbol
+              value.to_s.to_sym
+            when :datetime
+              DateTime.parse(value).to_time.utc unless value.nil?
+            else
+              value
+          end
+        end
+
+        def parse_options object, value, options
+          result = value
+          if result.nil?
+            result = options[:default]  unless options[:default].nil?
+          else
+            result = value[options[:regex], 1] unless options[:regex].nil?
+          end
+
+          result = object.send(options[:parser], result) unless options[:parser].nil?
+
+          result
+        end
+
+        def find_type object, field, polymorphic = nil
+          type_name = field.to_s.singularize.camelize
+          type_name = "#{object.send(polymorphic).to_s.camelize}#{type_name}" if polymorphic
+          type = type_name.safe_constantize
+          type = "#{object.class.to_s.deconstantize}::#{type_name}".constantize if type.nil?
+
+          type
+        end
+      end
+    end
+  end
+end

data/lib/dca/models/nokogiri_binder.rb

@@ -0,0 +1,43 @@
+module DCA
+  module Models
+    class NokogiriBinder
+      include BinderHelper
+
+      def self.parse object, content, params
+        if params[:association] == :one
+          parse_one object, content, params
+        elsif params[:association] == :many
+          parse_many object, content, params
+        end
+      end
+
+      def self.parse_one object, content, params
+        result = nil
+
+        options = params[:options] || {}
+        selector = options[:selector]
+        element = selector.nil? ? content : content.at_css(selector)
+        unless element.nil?
+          result = options[:attribute].nil? ? element.content : element[options[:attribute]]
+        end
+        result = self.parse_options object, result, options
+
+        convert result, params[:type]
+      end
+
+      def self.parse_many object, content, params
+        result = nil
+
+        options = params[:options] || {}
+        selector = options[:selector]
+        type = params[:type]
+        type = find_type object, params[:field], options[:polymorphic] if type.nil? || options[:polymorphic]
+
+        result = content.css(selector).map { |node| type.new.bind node } unless selector.nil?
+        result
+      end
+
+    end
+  end
+end
+

data/lib/dca/models/position.rb

@@ -0,0 +1,15 @@
+module DCA
+  module Models
+    class Position < BaseModel
+      attr_accessor :checksum, :published_at
+
+      validates_presence_of :base_id, :checksum
+
+      def initialize(*args)
+        # set instance variable id, need to place this attribute to hash, when id is not set.
+        @id = nil
+        super
+      end
+    end
+  end
+end

data/lib/dca/net.rb

@@ -0,0 +1 @@
+require File.expand_path('../net/browser_helper', __FILE__)

data/lib/dca/net/browser_helper.rb

@@ -0,0 +1,20 @@
+module DCA
+  module Net
+    module BrowserHelper
+      #def included(base)
+      #  base.extend ClassMethods
+      #end
+
+      #module ClassMethods
+      #end
+
+      def browser(name = :ff, profile = 'default')
+        @browser ||= Watir::Browser.new name, :profile => profile
+      end
+
+      def browser_close
+        @browser.close if @browser
+      end
+    end
+  end
+end

data/lib/dca/notifier.rb

@@ -0,0 +1,2 @@
+require File.expand_path('../notifier/notifier', __FILE__)
+require File.expand_path('../notifier/redis_notifier', __FILE__)

data/lib/dca/notifier/notifier.rb

@@ -0,0 +1,11 @@
+module DCA
+  class Notifier
+    def self.create config
+      @driver = "DCA::#{config[:driver]}Notifier".constantize.new config
+    end
+
+    def self.push object, event, options = {}
+      @driver.push object, event, options unless @driver.nil?
+    end
+  end
+end

data/lib/dca/notifier/redis/models/analyze_notify.rb

@@ -0,0 +1,12 @@
+module DCA
+  module Redis
+    class AnalyzeNotify < Ohm::Model
+      attribute :state # create, update, unchange, failed
+
+      index :state
+
+      counter :count
+      #reference :session, Session
+    end
+  end
+end

data/lib/dca/notifier/redis/models/failure_notify.rb

@@ -0,0 +1,8 @@
+module DCA
+  module Redis
+    class FailureNotify < Ohm::Model
+      attribute :message
+      attribute :stack
+    end
+  end
+end

data/lib/dca/notifier/redis/models/fetch_notify.rb

@@ -0,0 +1,15 @@
+module DCA
+  module Redis
+    class FetchNotify < Ohm::Model
+      attribute :state
+
+      index :state
+
+      counter :success
+      counter :failure
+      #attribute :message
+      #attribute :stack
+      #reference :session, Session
+    end
+  end
+end

data/lib/dca/notifier/redis/models/session.rb

@@ -0,0 +1,52 @@
+module DCA
+  module Redis
+    class Session < Ohm::Model
+      attribute :uid
+      attribute :created
+      attribute :project
+      attribute :area
+
+      index :uid
+      index :project
+      index :area
+
+      set :analyzed, DCA::Redis::AnalyzeNotify
+      set :fetched, DCA::Redis::FetchNotify
+      set :failures, DCA::Redis::FailureNotify
+
+      def validate
+        assert_present :uid
+      end
+
+      def analyze_state state
+        self.analyzed.find(:state => state).first
+      end
+
+      def fetch_state state
+        self.fetched.find(:state => state).first
+      end
+
+      def inc_analyze state
+        notify = self.analyzed.find(:state => state).first
+        if notify.nil?
+          notify = AnalyzeNotify.create(:state => state)
+          self.analyzed.add notify
+        end
+        notify.incr :count
+      end
+
+      def inc_fetch state, result
+        notify = self.fetched.find(:state => state).first
+        if notify.nil?
+          notify = FetchNotify.create(:state => state)
+          self.fetched.add notify
+        end
+        notify.incr result
+      end
+
+      def add_failure exception
+        self.failures.add FailureNotify.create(:message => exception.message, :stack => exception.backtrace)
+      end
+    end
+  end
+end