ac-summarization-utils 0.0.47.rc1

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,14 @@
+source "http://rubygems.org"            # Default source
+source "http://gems.kontera.com"
+
+gem "a15-worker"
+gem "redis"
+gem "mysql2"
+
+group :test do
+  gem "rspec", "1.1.11"
+  gem "rspec-core", "2.7.1"
+  gem "rspec-expectations", "2.7.0"
+  gem "rspec-mocks", "2.7.0"
+  gem "guard-rspec"
+end

data/README.md

@@ -0,0 +1,22 @@
+ac-summarization-utils gem
+==========================
+
+### Description ###
+
+This gem has all necessary utils to implement summarization workers:
+
+* lookup_cache - a class that implements various lookups in the DB with REDIS as its cache
+* validation_utils - utils that cehck if clicks and layer impressions are valid
+* summarization_utils - utils tjat summarize records by various summarization functions (currently - sum and count are supported)
+* results_dal - a class that handles insertion of summarization output to Mysql DB.
+
+
+### Unit tests ###
+
+Use the rspec framework for unit testing various utils.
+
+    $> bundle install
+    $> cd spec
+    $> rspec *.rb
+
+

data/Rakefile

@@ -0,0 +1,2 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks

data/ac-summarization-utils.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "ac-summarization-utils/version"
+
+Gem::Specification.new do |s|
+  s.name        = "ac-summarization-utils"
+  s.version     = ACSummarizationUtils::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Efrat Safanov"]
+  s.email       = ["efrat@kontera.com"]
+  s.homepage    = ""
+  s.summary     = %q{ac-summarization-utils files}
+  s.description = %q{A set of common classes and utilities for the ac-summarization workers}
+
+  s.rubyforge_project = "ac-summarization-utils"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+  s.add_dependency('redis')
+  s.add_dependency('mysql2')
+  s.add_dependency('a15-worker')
+end

data/ac-summarization-utils.iml

@@ -0,0 +1,28 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="RUBY_MODULE" version="4">
+  <component name="FacetManager">
+    <facet type="RailsFacetType" name="Ruby on Rails">
+      <configuration>
+        <RAILS_FACET_CONFIG_ID NAME="RAILS_FACET_SUPPORT_REMOVED" VALUE="false" />
+        <RAILS_FACET_CONFIG_ID NAME="RAILS_TESTS_SOURCES_PATCHED" VALUE="true" />
+        <RAILS_FACET_CONFIG_ID NAME="RAILS_FACET_APPLICATION_ROOT" VALUE="$MODULE_DIR$" />
+      </configuration>
+    </facet>
+  </component>
+  <component name="NewModuleRootManager" inherit-compiler-output="true">
+    <exclude-output />
+    <content url="file://$MODULE_DIR$">
+      <sourceFolder url="file://$MODULE_DIR$/test" isTestSource="true" />
+      <sourceFolder url="file://$MODULE_DIR$/spec" isTestSource="true" />
+    </content>
+    <orderEntry type="jdk" jdkName="RVM: ruby-1.9.3-p0 [summarization]" jdkType="RUBY_SDK" />
+    <orderEntry type="sourceFolder" forTests="false" />
+    <orderEntry type="library" scope="PROVIDED" name="a15-worker (v0.0.13, RVM: ruby-1.9.3-p0 [summarization]) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="amq-client (v0.9.2, RVM: ruby-1.9.3-p0 [summarization]) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="amq-protocol (v0.9.0, RVM: ruby-1.9.3-p0 [summarization]) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="amqp (v0.9.4, RVM: ruby-1.9.3-p0 [summarization]) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="bunny (v0.7.9, RVM: ruby-1.9.3-p0 [summarization]) [gem]" level="application" />
+    <orderEntry type="library" scope="PROVIDED" name="eventmachine (v0.12.10, RVM: ruby-1.9.3-p0 [summarization]) [gem]" level="application" />
+  </component>
+</module>
+

data/lib/ac-summarization-utils.rb

@@ -0,0 +1,8 @@
+require_relative 'ac-summarization-utils/record'
+require_relative 'ac-summarization-utils/errors_handler'
+require_relative 'ac-summarization-utils/summarization_utils'
+require_relative 'ac-summarization-utils/lookup_cache'
+require_relative 'ac-summarization-utils/results_dal'
+require_relative 'ac-summarization-utils/validation_utils'
+require_relative 'ac-summarization-utils/revenue_calc_utils'
+require_relative 'ac-summarization-utils/general_utils'

data/lib/ac-summarization-utils/errors_handler.rb

@@ -0,0 +1,77 @@
+require 'a15-worker'
+
+module ACSummarizationUtils
+
+  class ErrorsHandler
+    include A15::Worker
+    
+    def initialize(rabbit_db_host, exchange_input_errors, exchange_db_errors, error_key, message_type, original_rabbit_db_host=nil)
+      @hostname =`hostname`.strip
+      @error_key = error_key
+      @message_type = message_type
+      @rabbit_db_host = rabbit_db_host
+      input_error_queue_options = {
+        :host              => @rabbit_db_host,
+        :exchange_name        => exchange_input_errors
+        }
+      @input_error_producer = Producer.new(input_error_queue_options)
+      queries_error_queue_options = {
+        :host              => @rabbit_db_host,
+        :exchange_name        => exchange_db_errors
+        }
+      @queries_error_producer = Producer.new(queries_error_queue_options)
+      @original_rabbit_db_host = original_rabbit_db_host
+    end
+
+    def send_input_errors(logger, messages)
+      messages.each do |message|
+        publish_to_error_exchange(logger, @input_error_producer,message, @error_key)
+      end
+    end
+
+    def gen_and_send_input_errors(logger, messages, e)
+      print_error(logger, "Caught exception of type #{e.class} ,error message - #{e.message}, sending erroneous messages to a different queue")
+      messages.each do |message|
+        publish_to_error_exchange(logger, @input_error_producer,generate_error_message(message,e), @error_key)
+      end
+    end
+
+    def gen_and_send_db_errors(logger, messages, e)
+      print_error(logger, "Caught exception of type #{e.class} ,error message - #{e.message},queries - #{e.queries} - sending erroneous messages to a different queue")
+      queries_error_message = generate_error_message(messages,e)
+      queries_error_message[:queries] = e.queries
+      publish_to_error_exchange(logger, @queries_error_producer,queries_error_message, @error_key)
+    end
+
+    #generate an error message out of an input message
+    def generate_error_message(message,exception)
+      hostname =`hostname`.strip
+      rabbit_host = @original_rabbit_db_host ? @original_rabbit_db_host : @rabbit_db_host
+      return {:messages => message, :hostname => hostname, :key => @error_key, :rabbit_host => rabbit_host, :message_type => @message_type,  
+              :exception=>{:message=>exception.message,:backtrace=>exception.backtrace, :class=>exception.class}}
+    end
+
+    def handle_fatal(logger, e)
+      print_error(logger, "Caught fatal exception of type #{e.class} - #{e.message}, #{e.backtrace} - re-raising exception")
+      raise e
+    end
+
+    private
+    def publish_to_error_exchange(logger, error_producer,message, key)
+      begin
+        error_producer.publish(Base64.encode64(Zlib::Deflate.deflate(message.to_json, 9)), {:key => key})
+        logger.log(:info, "Published message to error queue. Message: #{message.inspect}")
+      rescue Exception => e
+        print_error(logger, "Unable to publish to errors exchange: producer - #{error_producer} - #{e.message}, #{e.backtrace} - rejecting messages back to queue")
+        raise e
+      end
+    end
+    
+    #print an error message both to logger and  to stderr
+    def print_error(logger,err_msg_str)
+      logger.log(:error,err_msg_str)
+      $stderr.puts err_msg_str
+    end
+
+  end
+end

data/lib/ac-summarization-utils/general_utils.rb

@@ -0,0 +1,25 @@
+#implements queries to DB with Redis cache (need to support pre-fetch)
+require 'set'
+
+module ACSummarizationUtils
+  module GeneralUtils
+
+    SUPPORTED_COUNTRIES = ['us','gb','it','fr','de','es','ca','in','au','ru','za'].to_set
+    SUPPORTED_LATIN_COUNTRIES = ['mx','bz','cr','gt','sv','hn','ni','pa','ar','bo','br','cl','co','ec','py','pe','uy','ve'].to_set
+    SUPPORTED_FAR_EAST = ['cn','hk','tw','sg','my','ph','jp'].to_set
+
+    def convert_country_code(country_code)
+      case
+      when SUPPORTED_LATIN_COUNTRIES.include?(country_code)
+        'latin_am'
+      when SUPPORTED_FAR_EAST.include?(country_code)
+        'f_east'
+      when SUPPORTED_COUNTRIES.include?(country_code)
+        country_code
+      else
+        'oth'
+      end
+    end
+  end
+end
+

data/lib/ac-summarization-utils/lookup_cache.rb

@@ -0,0 +1,480 @@
+#implements queries to DB with Redis cache (need to support pre-fetch)
+
+
+require 'mysql2'
+require 'redis'
+require 'time'
+require 'json'
+
+module ACSummarizationUtils
+
+  class LookupCache
+
+    ADGROUP_KEY_PREFIX = "k:adgroup"
+    ADGROUP_QUERY = "select adgroup_id from adc_ads ads where ads.id = %AD_ID%;"
+
+    CAMPAIGN_KEY_PREFIX = "k:campaign"
+    CAMPAIGN_QUERY = "select campaign_id from adc_adgroups adg join adc_ads ads on (ads.adgroup_id = adg.id)  where ads.id = %AD_ID%;"
+
+    ACCOUNT_KEY_PREFIX = "k:account"
+    ACCOUNT_QUERY = "select account_id from adc_campaigns cmp join
+                     adc_adgroups adg on (adg.campaign_id = cmp.id) join adc_ads ads on (ads.adgroup_id = adg.id) where ads.id = %AD_ID%;"
+
+    ADGROUP_TOPICS_KEY_PREFIX = "k:adgroup_topics"
+    ADGROUP_TOPICS_QUERY = "select Distinct(subTopic_id) as subtopic FROM adc_adgroups_subtopics aat JOIN adc_adgroup_settings aas
+                                   USING (adgroup_id) WHERE aat.adgroup_id = %ADGROUP_ID% AND aas.is_topical = 1"
+
+    ADGROUP_PUBLISHER_PUBGROUPS_KEY_PREFIX = "k:adgroup_publisher_pubgroups"
+    ADGROUP_PUBLISHER_PUBGROUPS_QUERY = "SELECT DISTINCT(PGP.publisherGroup_ID) AS pubgroup_id FROM PublisherGroupPlacement PGP JOIN adc_adgroups_pubgroups AAP ON (PGP.publisherGroup_ID = AAP.pubgroup_id) WHERE PGP.publisher_ID=%PUBLISHER_ID% AND AAP.adgroup_id=%ADGROUP_ID%"
+
+    ADVERTISER_SHARES_KEY_PREFIX = "k:advertiser_shares"
+    ADVERTISER_SHARES_QUERY = "SELECT advertiserShare,reserveShare FROM Advertiser WHERE advertiser_id = %ADVERTISER_ID%"
+    CAMPAIGN_SHARE_QUERY = "SELECT reserve/100 AS reserve FROM adc_campaign_financial_settings WHERE campaign_id = %CAMPAIGN_ID%"
+
+    PUBLISHER_PAYMENT_INFO_KEY_PREFIX = "k:publisher_payments_info"
+    PUBLISHER_PAYMENT_INFO_QUERY = "SELECT site_payment_type_id, rev_share, cpm FROM sites WHERE id = %PUBLISHER_ID%"
+
+    CAMPAIGN_CPM_PREFIX = "k:campaign_cpm"
+    CAMPAIGN_CPM_QUERY = "select CASE pricing_type_id WHEN 2 THEN pricing_rate ELSE 0 END AS cpm_rate from adc_campaign_financial_settings where campaign_id = %CAMPAIGN_ID%"
+
+    CAMPAIGN_CPC_PREFIX = "k:campaign_cpc"
+    CAMPAIGN_CPC_QUERY = "select CASE pricing_type_id WHEN 1 THEN pricing_rate ELSE 0 END AS cpc_rate from adc_campaign_financial_settings where campaign_id = %CAMPAIGN_ID%"
+
+    CAMPAIGN_PRICING_RATE_PREFIX = "k:campaign_pricing_rate"
+    CAMPAIGN_PRICING_RATE_QUERY = "select pricing_rate from adc_campaign_financial_settings where campaign_id = %CAMPAIGN_ID%"
+
+
+    CHANNEL_PUBLISHERS_PREFIX = "k:channel_publishers"
+    CHANNEL_PUBLISHERS_QUERY = "select id from sites where channel_enabled = 1;"
+
+    CAMPAIGN_PRICING_TYPE_PREFIX = "k:pricing_type_id"
+    CAMPAIGN_PRICING_TYPE_QUERY = "SELECT pricing_type_id FROM adc_campaign_financial_settings WHERE campaign_id = %CAMPAIGN_ID%"
+
+    ENGAGEMENT_AD_TYPE_PREFIX = "k:engagement_ad_type" 
+    ENGAGEMENT_AD_TYPE_QUERY = "SELECT id, ad_type_id FROM adc_ads WHERE id = %AD_ID%"
+
+    ENGAGEMENT_AD_VIDEO_PREFIX = "k:engagement_ad_video" 
+    ENGAGEMENT_AD_VIDEO_QUERY = "SELECT ad_id, ROUND(IF(rm_time IS NULL, 0, rm_time*1000)) as rm_time_millis FROM adc_ad_settings WHERE ad_id = %AD_ID%"
+
+    ENGAGEMENT_THRESHOLD_PREFIX = "k:engagement_threshold"
+    ENGAGEMENT_THRESHOLD_QUERY = "SELECT advanced_setting_value as engagement_qualifier FROM adc_campaign_advanced_settings WHERE campaign_id = %CAMPAIGN_ID% AND advanced_setting_type_id = 6"
+    
+    ENGAGEMENT_BY_PERCENTAGE_PREFIX = "k:engagment_by_percentage"
+    ENGAGEMENT_BY_PERCENTAGE_QUERY = "SELECT IF(engagement_report_by_percentage IS NULL, 0, engagement_report_by_percentage) as engagment_by_percentage FROM adc_ad_settings WHERE ad_id = %AD_ID%"
+
+    INIMAGE_KONTERA_PUBLISHER_ID_PREFIX = "k:inimage_publisher_id_map"
+    INIMAGE_KONTERA_PUBLISHER_ID_QUERY = "SELECT publisher_id FROM js_publisherparams LEFT JOIN js_params JP on js_param_id=JP.id where JP.name=\"luminate_account_id\" and paramValue = \"%INIMAGE_PUBLISHER_ID%\""
+
+    INIMAGE_PAYMENT_TYPE_PREFIX = "k:inimage_payment_type_id"
+    INIMAGE_PAYMENT_TYPE_QUERY = "select IFNULL(paramValue,defaultValue) as payment_type_id from (select A.name, A.defaultValue, B.publisher_id, B.paramValue from js_params as A left join (select * from js_publisherparams where publisher_id=\"%PUBLISHER_ID%\") as B  on A.id=B.js_param_id where name=\"inimage_payment_type_id\") as C"
+
+    INIMAGE_CPM_VALUE_PREFIX = "k:inimage_cpm_value"
+    INIMAGE_CPM_VALUE_QUERY = "select IFNULL(paramValue,defaultValue) as cpm from (select A.name, A.defaultValue, B.publisher_id, B.paramValue from js_params as A left join (select * from js_publisherparams where publisher_id=\"%PUBLISHER_ID%\") as B on A.id=B.js_param_id where name=\"inimage_cpm\") as C"
+
+    INIMAGE_REV_SHARE_PREFIX = "k:inimage_rev_share"
+    INIMAGE_REV_SHARE_QUERY = "select IFNULL(paramValue,defaultValue) as rev_share from (select A.name, A.defaultValue, B.publisher_id, B.paramValue from js_params as A left join (select * from js_publisherparams where publisher_id=\"%PUBLISHER_ID%\") as B on A.id=B.js_param_id where name=\"inimage_rev_share\") as C"
+
+    INIMAGE_PUBLISHER_PAYMENT_INFO_KEY_PREFIX =  "k:inimage_publisher_payments_info"
+
+    DUP_KEY_PREFIX = "k:dup"
+
+    class ResourcesError < RuntimeError; end
+    class InputError < RuntimeError; end
+
+    #throws exception if there is a resources connection problem
+    def initialize(redis_host, redis_port, redis_expiration, db_host, db_user, db_password, db_name)
+      @cache_expiration = redis_expiration.to_i
+      begin
+        @cache_client = Redis.connect({:host => redis_host, :port => redis_port.to_i })
+        @mysql_client = Mysql2::Client.new({:host => db_host, :username => db_user, :password => db_password, :database => db_name, :reconnect => true})
+      rescue Exception => e
+        @cache_client.client.disconnect if @cache_client and @cache_client.client and @cache_client.client.connected?
+        raise ResourcesError, e.message
+      end
+    end
+    def self.convert_url_to_host_and_port(options)
+        require "uri"
+        return options if !options[:url]
+        url = URI(options.delete(:url))
+        options[:host] = url.host
+        options[:port] = url.port
+        options
+    end
+    def self.connect(redis_options, lookup_db_options)
+        redis_options = convert_url_to_host_and_port(redis_options.dup)
+        new(redis_options[:host],redis_options[:port], redis_options[:expiration], lookup_db_options[:host], lookup_db_options[:user], lookup_db_options[:password], lookup_db_options[:name])
+    end
+
+    def get_adgroup_id(ad_id)
+      return get_field_by_ad_id(ad_id, ADGROUP_KEY_PREFIX, ADGROUP_QUERY, "adgroup_id")
+    end
+
+    def get_campaign_id(ad_id)
+      return get_field_by_ad_id(ad_id, CAMPAIGN_KEY_PREFIX, CAMPAIGN_QUERY, "campaign_id")
+    end
+
+    def get_campaign_pricing_type_id(campaign_id)
+      return get_field_by_campaign_id(campaign_id, CAMPAIGN_PRICING_TYPE_PREFIX, CAMPAIGN_PRICING_TYPE_QUERY, "pricing_type_id")
+    end
+
+    def get_account_id(ad_id)
+      return get_field_by_ad_id(ad_id, ACCOUNT_KEY_PREFIX, ACCOUNT_QUERY, "account_id")
+    end
+
+    def get_engagement_type(ad_id)
+      return get_field_by_ad_id(ad_id, ENGAGEMENT_AD_TYPE_PREFIX, ENGAGEMENT_AD_TYPE_QUERY, "ad_type_id")
+    end
+
+    def get_engagement_video_length(ad_id)
+      return get_field_by_ad_id(ad_id, ENGAGEMENT_AD_VIDEO_PREFIX, ENGAGEMENT_AD_VIDEO_QUERY, "rm_time_millis")
+    end
+    
+    def get_is_engagement_by_percentage(ad_id)
+    	return get_field_by_ad_id(ad_id, ENGAGEMENT_BY_PERCENTAGE_PREFIX, ENGAGEMENT_BY_PERCENTAGE_QUERY, "engagment_by_percentage")
+    end
+
+    def get_publisher_id_by_inimage_id(inimage_publisher_id)
+      return get_publisher_id_by_inimage_id_helper(inimage_publisher_id, INIMAGE_KONTERA_PUBLISHER_ID_PREFIX, INIMAGE_KONTERA_PUBLISHER_ID_QUERY, "publisher_id")
+    end
+
+    def get_engagement_threshold(campaign_id)
+      begin
+        result = get_field_by_campaign_id(campaign_id, ENGAGEMENT_THRESHOLD_PREFIX, ENGAGEMENT_THRESHOLD_QUERY, "engagement_qualifier")
+	return JSON.parse(result)["value"].to_i if result
+      rescue => e
+      end
+      return nil
+    end
+
+    #deprecated: moved  to summarization-utils
+    #raises an exception if cannot parse date
+    def get_date(date)
+      return nil if date.nil?
+      begin
+        time_object =  date.is_a?(Fixnum) ? Time.at(date) : Time.parse(date)
+        return time_object.strftime("%Y-%m-%d")
+      rescue Exception => e
+        raise InputError, "Unable to parse time string: #{date} - message: #{e.message}"
+      end
+    end
+
+    def get_relevant_topic_ids(page_topic_ids, adgroup_id)
+      return [] if page_topic_ids.nil? or adgroup_id.nil?
+      topics_array = page_topic_ids.is_a?(Array) ? page_topic_ids : page_topic_ids.split(/(\+| )/)
+      return [] if topics_array.nil? or topics_array.empty?
+      adgroup_topics_ids = get_topic_ids(adgroup_id)
+      return [] if adgroup_topics_ids.nil? or adgroup_topics_ids.empty?
+      topics_array.map! { |t| t.to_i}
+      adgroup_topics_ids.map! {|t| t.to_i}
+      return topics_array & adgroup_topics_ids
+    end
+
+    def get_pubgroup_ids(adgroup_id,publisher_id)
+      return [] if publisher_id.nil? or adgroup_id.nil?
+      adgroup_publisher_pubgroup_ids = get_adgroup_publisher_pubgroup_ids(adgroup_id,publisher_id)
+      return [] if adgroup_publisher_pubgroup_ids.nil? or adgroup_publisher_pubgroup_ids.empty?
+      return adgroup_publisher_pubgroup_ids.map{|pg| pg.to_i}
+    end
+
+    def get_and_set_duplicate_value(key,expiry_secs)
+      res = lookup_in_cache("#{DUP_KEY_PREFIX}:#{key}")
+      return res if res
+      add_to_cache("#{DUP_KEY_PREFIX}:#{key}",true, expiry_secs)
+      return nil
+    end
+
+    def clear_duplicate_value(key)
+      delete_from_cache("#{DUP_KEY_PREFIX}:#{key}")
+    end
+
+    def get_advertiser_shares(advertiser_id,campaign_id=nil)
+      return {} if advertiser_id.nil?
+      advertiser_shares = get_advertisers_shares_info(advertiser_id,campaign_id)
+      return advertiser_shares
+    end
+
+    def get_publisher_payment_info(publisher_id)
+      return {} if publisher_id.nil?
+      publisher_payment_info = get_publisher_payment_data(publisher_id)
+      return publisher_payment_info
+    end
+
+    def get_inimage_publisher_payment_info(publisher_id)
+      return {} if publisher_id.nil?
+      publisher_payment_info = get_inimage_publisher_payment_data(publisher_id)
+      return publisher_payment_info
+    end
+
+    def get_campaign_cpm(campaign_id)
+      return campaign_id.nil? ? 0.0 : get_field_by_campaign_id(campaign_id, CAMPAIGN_CPM_PREFIX, CAMPAIGN_CPM_QUERY, "cpm_rate")
+    end
+
+    def get_campaign_cpc(campaign_id)
+      return campaign_id.nil? ? 0.0 : get_field_by_campaign_id(campaign_id, CAMPAIGN_CPC_PREFIX, CAMPAIGN_CPC_QUERY, "cpc_rate")
+    end
+
+    def get_campaign_pricing_rate(campaign_id)
+      get_field_by_campaign_id(campaign_id, CAMPAIGN_PRICING_RATE_PREFIX, CAMPAIGN_PRICING_RATE_QUERY, "pricing_rate")
+    end
+
+    def get_channel_publishers()
+      return get_channel_publishers_data()
+    end
+
+    private
+
+    def get_topic_ids(adgroup_id)
+      cache_result = get_adgroup_topics_from_cache(adgroup_id)
+      return cache_result if cache_result
+      db_result = get_adgroup_topics_from_db(adgroup_id)
+      add_to_cache("#{ADGROUP_TOPICS_KEY_PREFIX}:#{adgroup_id}", db_result.to_json)
+      return db_result
+    end
+
+    def get_adgroup_publisher_pubgroup_ids(adgroup_id,publisher_id)
+      cache_result = get_adgroup_publisher_pubgroups_from_cache(adgroup_id,publisher_id)
+      return cache_result if cache_result
+      db_result = get_adgroup_publisher_pubgroups_from_db(adgroup_id,publisher_id)
+      add_to_cache("#{ADGROUP_PUBLISHER_PUBGROUPS_KEY_PREFIX}:#{adgroup_id}:#{publisher_id}",db_result.to_json)
+      return db_result
+    end
+
+    def get_advertisers_shares_info(advertiser_id,campaign_id=nil)
+      key_suffix = campaign_id.nil? ? ":#{advertiser_id}" : ":#{advertiser_id}:#{campaign_id}"
+      cache_result = get_advertiser_shares_from_cache(key_suffix)
+      return cache_result if cache_result
+      db_result = get_advertiser_shares_from_db(advertiser_id,campaign_id)
+      add_to_cache("#{ADVERTISER_SHARES_KEY_PREFIX}#{key_suffix}",db_result.to_json)
+      return db_result
+    end
+
+    def get_publisher_payment_data(publisher_id)
+      cache_result = get_publisher_payment_info_from_cache(publisher_id)
+      return cache_result if cache_result
+      db_result = get_publisher_payment_info_from_db(publisher_id)
+      add_to_cache("#{PUBLISHER_PAYMENT_INFO_KEY_PREFIX}:#{publisher_id}",db_result.to_json)
+      return db_result
+    end
+
+    def get_field_by_ad_id(ad_id, cache_key_prefix, db_query, db_field)
+      return nil if ad_id.nil?
+      cache_key = "#{cache_key_prefix}:#{ad_id}"
+      cache_result = lookup_in_cache(cache_key)
+      return cache_result if cache_result
+      db_result = lookup_in_db(db_query, ad_id, db_field)
+      if db_result
+        add_to_cache(cache_key, db_result)
+        return db_result
+      else
+        raise InputError, "Can't get #{db_field} from DB for ad_id: #{ad_id}"
+      end
+    end
+
+    def get_adgroup_topics_from_cache(adgroup_id)
+      res = lookup_in_cache("#{ADGROUP_TOPICS_KEY_PREFIX}:#{adgroup_id}")
+      return res.nil? ? nil : JSON.parse(res)
+    end
+
+    def get_adgroup_topics_from_db(adgroup_id)
+      query = ADGROUP_TOPICS_QUERY.gsub("%ADGROUP_ID%", "#{adgroup_id}")
+      topic_ids = []
+      begin
+        result = @mysql_client.query(query)
+        return [] if result.nil? or result.size == 0
+        result.each do |row|
+          topic_ids << row["subtopic"] unless row["subtopic"].nil?
+        end
+      rescue Mysql2::Error => e
+        raise ResourcesError, "Got MysqL exception in lookup for query - '#{query}'. Message: #{e.message}"
+      end
+      return topic_ids
+    end
+
+    def get_adgroup_publisher_pubgroups_from_cache(adgroup_id,publisher_id)
+      res = lookup_in_cache("#{ADGROUP_PUBLISHER_PUBGROUPS_KEY_PREFIX}:#{adgroup_id}:#{publisher_id}")
+      return res.nil? ? nil : JSON.parse(res)
+    end
+
+    def get_adgroup_publisher_pubgroups_from_db(adgroup_id,publisher_id)
+      query = ADGROUP_PUBLISHER_PUBGROUPS_QUERY.gsub("%ADGROUP_ID%", "#{adgroup_id}").gsub("%PUBLISHER_ID%","#{publisher_id}")
+      pubgroup_ids = []
+      begin
+        result = @mysql_client.query(query)
+        return [-1] if result.nil? or result.size == 0
+        result.each do |row|
+          pubgroup_ids << row["pubgroup_id"] unless row["pubgroup_id"].nil?
+        end
+      rescue Mysql2::Error => e
+        raise ResourcesError, "Got MysqL exception in lookup for query - '#{query}'. Message: #{e.message}"
+      end
+      return pubgroup_ids
+    end
+
+    def get_advertiser_shares_from_cache(key_suffix)
+      res = lookup_in_cache("#{ADVERTISER_SHARES_KEY_PREFIX}#{key_suffix}")
+      return res.nil? ? nil : JSON.parse(res)
+    end
+
+    def get_advertiser_shares_from_db(advertiser_id,campaign_id=nil)
+      camp_query = CAMPAIGN_SHARE_QUERY.gsub("%CAMPAIGN_ID%","#{campaign_id}") if !campaign_id.nil?
+      shares = {"reserve_share" => 0.0, "adv_share" => 0.0, "campaign_reserve" => 0.0}
+      begin
+        adv_result = @mysql_client.query(ADVERTISER_SHARES_QUERY.gsub("%ADVERTISER_ID%","#{advertiser_id}"))
+        camp_result = @mysql_client.query(camp_query) if !campaign_id.nil?
+        return shares if ((adv_result.nil? or adv_result.size == 0) and (camp_result.nil? or camp_result.size == 0))
+        if (!adv_result.first.nil?)
+         shares["adv_share"] = adv_result.first["advertiserShare"].to_f  unless adv_result.first["advertiserShare"].nil?
+         shares["reserve_share"] = adv_result.first["reserveShare"].to_f unless adv_result.first["reserveShare"].nil?
+        end
+        shares["campaign_reserve"] = camp_result.first["reserve"].to_f unless (camp_result.nil? || camp_result.first.nil? || camp_result.first["reserve"].nil?)
+      end
+
+      return shares
+    end
+
+    def get_publisher_payment_info_from_cache(publisher_id)
+      res = lookup_in_cache("#{PUBLISHER_PAYMENT_INFO_KEY_PREFIX}:#{publisher_id}")
+      return res.nil? ? nil : JSON.parse(res)
+    end
+
+    def get_publisher_payment_info_from_db(publisher_id)
+      publisher_payment_info = {"payment_type" => 1, "cpm" => 0.0, "rev_share" => 0.0}
+      res = @mysql_client.query(PUBLISHER_PAYMENT_INFO_QUERY.gsub("%PUBLISHER_ID%","#{publisher_id}"))
+      return publisher_payment_info if (res.nil? or res.size == 0)
+      publisher_payment_info["payment_type"] = res.first["site_payment_type_id"]
+      publisher_payment_info["rev_share"] = res.first["rev_share"].to_f
+      publisher_payment_info["cpm"] = res.first["cpm"].to_f
+      return publisher_payment_info
+    end
+
+	def get_field_by_campaign_id(campaign_id, cache_key_prefix, db_query, db_field)
+      cache_key = "#{cache_key_prefix}:#{campaign_id}"
+      cache_result = lookup_in_cache(cache_key)
+      return cache_result if cache_result
+      db_result = get_field_by_campaign_from_db(db_query, campaign_id, db_field)
+      if db_result
+        add_to_cache(cache_key, db_result)
+        return db_result
+      else
+        raise InputError, "Can't get #{db_field} from DB for campaign_id: #{campaign_id}"
+      end
+    end
+
+    def get_channel_publishers_data()
+      cache_key = "#{CHANNEL_PUBLISHERS_PREFIX}"
+      cache_result = lookup_in_cache(cache_key)
+      return cache_result if cache_result
+      channel_publishers = []
+      begin
+        result = @mysql_client.query(CHANNEL_PUBLISHERS_QUERY)
+        return [] if result.nil? or result.size == 0
+        result.each do |row|
+          channel_publishers << row["id"] unless row["id"].nil?
+        end
+      rescue Mysql2::Error => e
+        raise ResourcesError, "Got MysqL exception in lookup for query - '#{CHANNEL_PUBLISHERS_QUERY}'. Message: #{e.message}"
+      end
+      add_to_cache(cache_key, channel_publishers )
+      return channel_publishers
+    end
+
+    def add_to_cache(key, value, expiry = @cache_expiration)
+      begin
+        @cache_client.setex(key, expiry, value)
+      rescue Exception => e
+        raise ResourcesError, "Got Redis exception in adding to cache for key - '#{key}' with value: '#{value}'. Message: #{e.message}"
+      end
+    end
+  
+    def delete_from_cache(key)
+      begin
+        @cache_client.del(key)
+      rescue Exception => e
+        raise ResourcesError, "Got Redis exception in deleting from cache - '#{key}'. Message: #{e.message}"
+      end
+    end
+
+    def lookup_in_cache(key)
+      begin
+        return @cache_client.get(key)
+      rescue Exception => e
+        raise ResourcesError, "Got Redis exception in getting key - '#{key}'. Message: #{e.message}"
+      end
+    end
+
+    def lookup_in_db(query, val, res_field_name)
+      full_query = query.gsub("%AD_ID%", "#{val}")
+      begin
+        result = @mysql_client.query(full_query)
+        return nil if result.nil? or result.size == 0
+        return result.first[res_field_name]
+      rescue Mysql2::Error => e
+        raise ResourcesError, "Got MysqL exception in lookup for query - '#{full_query}'. Message: #{e.message}"
+      end
+    end
+
+    def get_field_by_campaign_from_db(query, campaign_id, res_field_name)
+      full_query = query.gsub("%CAMPAIGN_ID%", "#{campaign_id}")
+      begin
+        result = @mysql_client.query(full_query)
+        return nil if result.nil? or result.size == 0
+        return result.first[res_field_name]
+      rescue Mysql2::Error => e
+        raise ResourcesError, "Got MysqL exception in lookup for query - '#{full_query}'. Message: #{e.message}"
+      end
+    end
+
+    def get_publisher_id_by_inimage_id_from_db(query, inimage_publisher_id, res_field_name)
+      full_query = query.gsub("%INIMAGE_PUBLISHER_ID%", "#{inimage_publisher_id}")
+      begin
+        result = @mysql_client.query(full_query)
+        return nil if result.nil? or result.size == 0
+        return result.first[res_field_name]
+      rescue Mysql2::Error => e
+        raise ResourcesError, "Got MysqL exception in lookup for query - '#{full_query}'. Message: #{e.message}"
+      end
+    end
+
+    def get_publisher_id_by_inimage_id_helper(inimage_publisher_id, cache_key_prefix, db_query, db_field)
+      return nil if inimage_publisher_id.nil?
+      cache_key = "#{cache_key_prefix}:#{inimage_publisher_id}"
+      cache_result = lookup_in_cache(cache_key)
+      return cache_result if cache_result
+      db_result = get_publisher_id_by_inimage_id_from_db(db_query, inimage_publisher_id, db_field)
+      if db_result
+        add_to_cache(cache_key, db_result)
+        return db_result
+      else
+        raise InputError, "Can't get #{db_field} from DB for inimage_publisher_id: #{inimage_publisher_id}"
+      end
+    end
+
+    def get_inimage_publisher_payment_info_from_cache(publisher_id)
+      res = lookup_in_cache("#{INIMAGE_PUBLISHER_PAYMENT_INFO_KEY_PREFIX}:#{publisher_id}")
+      return res.nil? ? nil : JSON.parse(res)
+    end
+
+    def get_inimage_publisher_payment_info_from_db(publisher_id)
+      publisher_payment_info = {"payment_type" => 1, "cpm" => 0.0, "rev_share" => 0.0}
+      res_payment_type_id = @mysql_client.query(INIMAGE_PAYMENT_TYPE_QUERY.gsub("%PUBLISHER_ID%","#{publisher_id}"))
+      res_cpm_value = @mysql_client.query(INIMAGE_CPM_VALUE_QUERY.gsub("%PUBLISHER_ID%","#{publisher_id}"))
+      res_rev_share = @mysql_client.query(INIMAGE_REV_SHARE_QUERY.gsub("%PUBLISHER_ID%","#{publisher_id}"))
+      return publisher_payment_info if (res_payment_type_id.nil? or res_payment_type_id.size == 0)
+      publisher_payment_info["payment_type"] = res_payment_type_id.first["payment_type_id"].to_i
+      publisher_payment_info["rev_share"] = (res_rev_share.nil? or res_rev_share.size == 0) ? 0.0 : res_rev_share.first["rev_share"].to_f
+      publisher_payment_info["cpm"] = (res_cpm_value.nil? or res_cpm_value.size == 0) ? 0.0 : res_cpm_value.first["cpm"].to_f
+
+      return publisher_payment_info
+    end
+
+    def get_inimage_publisher_payment_data(publisher_id)
+      cache_result = get_inimage_publisher_payment_info_from_cache(publisher_id)
+      return cache_result if cache_result
+      db_result = get_inimage_publisher_payment_info_from_db(publisher_id)
+      add_to_cache("#{INIMAGE_PUBLISHER_PAYMENT_INFO_KEY_PREFIX}:#{publisher_id}",db_result.to_json)
+      return db_result
+    end
+
+  end
+
+end