dailyrep 1.0.0

data/lib/dailyrep/IBrowser.rb

@@ -0,0 +1,77 @@
+require 'open-uri'
+require 'nokogiri'
+require 'json'
+require 'net/http'
+
+require_relative 'Notificator'
+require_relative 'IndexWriter'
+require_relative 'Dbops'
+require_relative 'Trackable'
+
+module DailyRep
+  class IBrowser
+    include Dbops
+    include Trackable
+    attr_reader :entity
+
+    @@notificators = Notificator.get_clients
+    @@index_writer = IndexWriter.new
+    @@app_cycle_methods = ['process', 'write_to_db', 'web_reload', 'notify']
+
+    def initialize *inputs
+      @entity = self.class.to_s.downcase.split('::').last
+    end
+
+    def push_note note
+      @@notificators.each { |client|
+        client.push_note(note)
+      }
+      # @@client.push_note(nil, @@note_title,  note)
+    end
+
+    def set_html entity, values, push=0
+      @@index_writer.set_html entity, values, push
+    end
+
+    def set_html_with_diff entity, scope, diff, reset=false
+      @@index_writer.set_html_with_diff entity, scope, diff, reset
+    end
+
+    def self.write_html
+      @@index_writer.write_html
+    end
+
+
+    def get_source_http source, val_scope
+      page = Nokogiri::HTML(open(source.to_s))
+      page.css(val_scope)
+    end
+
+    def get_source_json source
+      uri = URI(source)
+      out_json = Net::HTTP.get(uri)
+      JSON.parse(out_json)
+    end
+
+    def method_missing(meth, *args, &block)
+      if @@app_cycle_methods.include?(meth.to_s)
+        begin
+          self.log_msg meth.to_s
+          block.call
+          rescue Exception => e
+            log_error e
+        end
+      else
+        super
+      end
+    end
+
+    def run
+      self.process
+      self.write_to_db if Configer.is_phase_enabled?('write_to_db')
+      self.notify if Configer.is_phase_enabled?('notify')
+      self.web_reload if Configer.is_phase_enabled?('web_reload')
+    end
+
+  end
+end

data/lib/dailyrep/IndexWriter.rb

@@ -0,0 +1,55 @@
+module DailyRep
+  class IndexWriter
+
+    def initialize(index_path=Configer.index_path)
+      @out_html_path = index_path.to_s
+      @out_html = Nokogiri::HTML(File.open(@out_html_path))
+    end
+
+    def write_html
+      @out_html.css("#refresh")[0].content = Time.now
+      File.open(@out_html_path , 'w') { |file| file.write(@out_html.to_html) }
+    end
+
+    def set_html entity, params, notif=0
+      entity = entity.downcase
+      params.each do |key, val|
+        @out_html.css("##{entity}_#{key}")[0].content = val
+      end
+      @out_html.css("##{entity}_refresh")[0].content = Time.now
+      @out_html.css("##{entity}_notif")[0].content = Time.now if notif == 1
+    end
+
+    def set_html_with_diff entity, scope, diff, reset=false
+      entity = entity.downcase
+        target_div = @out_html.css(scope + " .list-group .list-group-item")
+        new_root_node = Nokogiri::XML::Node.new('div', @out_html)
+        new_root_node['class'] = 'list-group'
+
+
+        diff.each do |diff_el|
+          target_div.each { |node|
+            if diff_el[0] == '+' && node['href'].include?(diff_el[2].first[0]) then
+              target_div.delete(node)
+            end
+          }
+        end
+
+        new_root_node.add_child(target_div.unlink) if not reset
+
+        diff.each do |diff_el|
+          if diff_el[0] == '-' then
+            nodeel = Nokogiri::XML::Node.new('a', @out_html)
+            nodeel['href'] = diff_el[2].first[0]
+            nodeel['class'] = 'list-group-item'
+            nodeel.content = diff_el[2].first[1]
+            new_root_node.add_child(nodeel)
+          end
+        end
+
+        @out_html.at_css(scope + " .list-group").replace(new_root_node)
+
+      @out_html.css("##{entity}_refresh")[0].content = Time.now
+    end
+  end
+end

data/lib/dailyrep/Loadconfig.rb

@@ -0,0 +1,135 @@
+
+class String
+  def to_bool
+    return true if self == true || self =~ (/(true|t|yes|y|1)$/i)
+    return false if self == false || self.blank? || self =~ (/(false|f|no|n|0)$/i)
+    raise ArgumentError.new("invalid value for Boolean: \"#{self}\"")
+  end
+end
+
+ActiveRecord::Base.establish_connection(
+  :adapter  => 'sqlite3',
+  :database => DATABASE,
+  :pool     => 5,
+  :timeout  => 5000
+)
+
+def generate_database
+  ActiveRecord::Schema.define(version: 201502161646) do
+    create_table "Histories", force: false do |t|
+      t.text     "entity"
+      t.text     "parameter"
+      t.text     "val"
+      t.datetime "created_at"
+      t.datetime "notified"
+    end
+  end
+end
+
+tables = ActiveRecord::Base.connection.execute(
+    "SELECT name FROM sqlite_master WHERE type='table' and name = 'Histories'; "
+)
+
+if tables.empty?
+  generate_database
+else
+  generate_database unless tables[0]["name"] == 'Histories'
+end
+
+
+class History < ActiveRecord::Base
+end
+
+module DailyRep
+
+  START_TIME = Time.now
+
+  #constants
+  P_GO_FILE_PATH    = 'GoFilePath'
+  P_PUSH_CLIENTS    = 'Push clients'
+  P_PUSH_NOTE_TITLE = 'Push title'
+  P_RAISE_ERRORS    = 'Raise errors'
+
+  PA_ENTITIES   = 'Entities'
+  PA_APP_CYCLE  = 'app_cycle'
+  P_INDEX_PATH  = 'Index path'
+  P_NOTIF_DELTA = 'notification delta'
+  P_ENABLED     = 'enabled'
+  P_TIME_FRAME  = 'time frame'
+  P_SOURCE      = 'source'
+  P_SEARCH_STRS = 'searchStr'
+
+
+
+
+
+  class Configer
+    def self.set_index_file file
+      @@config_json[P_INDEX_PATH] = file
+    end
+
+    File.open(CONFIG_FILE) {|file| @@config_json = JSON.parse(file.read) }
+
+    def self.index_path
+      @@config_json[P_INDEX_PATH]
+    end
+
+    def self.push_note_title
+      @@config_json[P_PUSH_NOTE_TITLE]
+    end
+
+    def self.raise_error?
+      @@config_json[P_RAISE_ERRORS].to_bool
+    end
+
+    def self.exctract_entity_params entity, parameter_name
+      @@config_json[PA_ENTITIES][entity][parameter_name]
+    end
+
+    def self.is_phase_enabled? phase
+      @@config_json[PA_APP_CYCLE][phase].to_bool
+    end
+    #+
+    def self.is_entity_enabled? entity
+      time_arr = @@config_json[PA_ENTITIES][entity][P_TIME_FRAME].split('..').map { |el|
+        el.to_i
+      }
+      (time_arr[0]..time_arr[1]).include?(Time.now.hour) &&
+        @@config_json[PA_ENTITIES][entity][P_ENABLED].to_bool
+    end
+    #+
+    def self.enabled_entities
+      @@config_json[PA_ENTITIES].keys.map { |entity|
+        entity.downcase.capitalize if is_entity_enabled?(entity)
+      }
+    end
+    #+
+    def self.get_client_tokens
+      @@config_json[P_PUSH_CLIENTS].values
+    end
+
+    # def self.include_search_strs? entity
+    #   result = false
+    #   result = true if @@config_json[PA_ENTITIES][entity][P_SEARCH_STRS]
+    #   return result
+    # end
+
+    def self.get_search_strs entity
+      @@config_json[PA_ENTITIES][entity][P_SEARCH_STRS]
+    end
+
+    def self.method_missing(meth, *args, &block)
+      method_inputs = meth.to_s.split('_')
+      out_param = ''
+      @@config_json[PA_ENTITIES].keys.each { |key|
+        if method_inputs[0] == key.downcase
+          out_param = self.exctract_entity_params method_inputs[0], method_inputs[1]
+        end
+      }
+      super if not @@config_json[PA_ENTITIES].keys.include?(method_inputs[0])
+      return out_param
+    end
+
+  end
+
+end

data/lib/dailyrep/Notificator.rb

@@ -0,0 +1,23 @@
+require 'washbullet'
+module DailyRep
+
+  class Notificator
+    def self.get_clients
+      result = []
+      result = Configer.get_client_tokens.map { |token|
+        new token
+      }
+    end
+
+    def initialize(push_token, note_title=Configer.push_note_title)
+        @client = Washbullet::Client.new(push_token)
+        @note_title = note_title
+    end
+
+    def push_note note
+      @client.push_note(nil, @note_title,  note)
+    end
+  end
+
+
+end

data/lib/dailyrep/Trackable.rb

@@ -0,0 +1,14 @@
+module DailyRep
+  module Trackable
+    def log_msg entity, add_param='', phase
+      p "#{entity} (#{add_param}), stage: #{phase} at #{Time.now}"
+    end
+
+    def log_error e
+      # p "    ERROR!: #{e.message}"
+      # p "    ERROR_MSG: #{e.backtrace.inspect}"
+      raise
+      # if Configer.raise_error?
+    end
+  end
+end

data/lib/dailyrep/entities/Doujob.rb

@@ -0,0 +1,83 @@
+require_relative '../Dbops'
+
+module DailyRep
+  module Entities
+
+
+    class Doujob < IBrowser
+      attr_reader  :diff
+      include Dbops
+
+      def self.create
+          result = []
+          result = Configer.get_search_strs('doujob').map { |str|
+            new str
+          }
+      end
+
+      def initialize search, source=Configer.doujob_source
+        super
+        @search=search.gsub(' ','+')
+        @dou_search_path = source + @search
+        @vacancy_scope = 'div#vacancyListId li'
+        @parameter = 'title-href'
+        @diff =[]
+      end
+
+      def log_msg phase
+        super @entity, @search, phase
+      end
+
+      #overwriting
+      def process
+        super  do
+          out_set = get_source_http @dou_search_path, @vacancy_scope
+          @out_vac = out_set.map do |a|
+            [a.css('.title .vt').attribute("href").text, "#{a.css('.company').text} : #{a.css('.title .vt').text}"]
+          end
+          out_hash = @out_vac.map { |el|
+            {el[0] => el[1]}
+          }
+          hist_hash = read_hist_hash
+          @diff = HashDiff.diff(out_hash, hist_hash)
+        end
+      end
+
+      #overwriting
+      def notify
+        super do
+          @diff.each { |lv1|
+            push_note("New vacancy at DOU: #{lv1[2]} - #{lv1[3]} ") if lv1[0] == "-"
+            push_note("DOU vacancy has gone: #{lv1[2]} - #{lv1[3]} ") if lv1[0] == "+"
+          }
+        end
+      end
+      #overwriting
+      def write_to_db
+        super do
+          @out_vac.each { |vac|
+            write_hist @entity, @parameter, "#{vac[0]}=>#{vac[1]}"
+          }
+        end
+      end
+      #overwriting
+      def web_reload
+        super do
+          set_html_with_diff @entity, '#doujobs', @diff
+        end
+      end
+
+      private
+      def read_hist_hash
+        maxtime = History.where(entity: @entity, parameter: @parameter).where("created_at > ?", Time.now - 6.hours).maximum("created_at")
+        maxtime||= Time.now
+        maxtime = maxtime.to_datetime
+        History.select("val").where(entity: @entity, parameter: @parameter).where("created_at > ?", maxtime - 1.minute).map { |row|
+          el_ar = row.val.split('=>')
+          {el_ar[0] => el_ar[1]}
+        }
+      end
+
+    end
+  end
+end

data/lib/dailyrep/entities/Kinozal.rb

@@ -0,0 +1,87 @@
+#encoding: utf-8
+require 'active_support/core_ext'
+module DailyRep
+  module Entities
+    class Kinozal < IBrowser
+      attr_reader :out_hash_clean, :to_notification
+
+      def self.create
+          result = []
+          result = Configer.get_search_strs('kinozal').map { |str|
+            new str
+          }
+      end
+
+      def initialize  movie, source=Configer.kinozal_source, filters=Configer.kinozal_filters
+        super
+        @movie = movie
+        @search_path = source+@movie.gsub(' ','+') + filters
+        @scope = 'td.nam  a'
+        @dirty_words = ['тизер', 'трейлер']
+        @out_hash_clean = {}
+      end
+
+      def log_msg phase
+        super @entity, @movie, phase
+      end
+
+      def process
+        super do
+          out_set = get_source_http @search_path, @scope
+          out_hash_dirty = {}
+          out_set.each { |el|
+            out_hash_dirty.merge!({ el['href'] => el.text.split(' / ')})
+          }
+          out_hash_dirty.each { |key, val|
+            exclude = 0
+            @dirty_words.each { |word|
+              if val[0].mb_chars.downcase.include?(word) then
+                exclude = 1
+                break
+              end
+            }
+            @out_hash_clean.merge!({key => val}) if exclude == 0
+          }
+        end
+      end
+
+      def write_to_db
+        super do
+          @to_notification = {}
+          @out_hash_clean.each { |key, val|
+            id = key.split('id=')[1]
+            if !check_row_exists(@entity, @movie, id) then
+              write_hist(@entity, @movie, id, 1)
+              @to_notification.merge!({key => [val[0], val.last]})
+            end
+          }
+        end
+      end
+
+      def notify
+        super do
+          @to_notification.each { |key, val|
+            note = "Kinozal
+            Movie appears: #{val[0]}
+            Quality: #{val.last}
+            Link: #{key}"
+            push_note note
+          }
+        end
+      end
+
+
+      def web_reload
+        super do
+          i = 0
+          diff = @to_notification.map { |key, val|
+            i = i+1
+            ['-', i , {'http://kinozal.tv/'+ key => val.join(' : ')} ]
+          }
+          set_html_with_diff @entity, '#kinozal',  diff, true
+        end
+      end
+
+    end
+  end
+end