ac-summarization-utils 0.0.47.rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +4 -0
- data/Gemfile +14 -0
- data/README.md +22 -0
- data/Rakefile +2 -0
- data/ac-summarization-utils.gemspec +24 -0
- data/ac-summarization-utils.iml +28 -0
- data/lib/ac-summarization-utils.rb +8 -0
- data/lib/ac-summarization-utils/errors_handler.rb +77 -0
- data/lib/ac-summarization-utils/general_utils.rb +25 -0
- data/lib/ac-summarization-utils/lookup_cache.rb +480 -0
- data/lib/ac-summarization-utils/record.rb +40 -0
- data/lib/ac-summarization-utils/results_dal.rb +189 -0
- data/lib/ac-summarization-utils/revenue_calc_utils.rb +41 -0
- data/lib/ac-summarization-utils/summarization_utils.rb +119 -0
- data/lib/ac-summarization-utils/validation_utils.rb +93 -0
- data/lib/ac-summarization-utils/version.rb +3 -0
- data/spec/errors_handler_spec.rb +52 -0
- data/spec/general_utils_spec.rb +35 -0
- data/spec/lookup_cache_spec.rb +956 -0
- data/spec/results_dal_spec.rb +349 -0
- data/spec/revenue_calc_utils_spec.rb +125 -0
- data/spec/summarization_utils_spec.rb +210 -0
- data/spec/validation_utils_spec.rb +328 -0
- metadata +116 -0
data/.gitignore
ADDED
data/Gemfile
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
source "http://rubygems.org" # Default source
|
2
|
+
source "http://gems.kontera.com"
|
3
|
+
|
4
|
+
gem "a15-worker"
|
5
|
+
gem "redis"
|
6
|
+
gem "mysql2"
|
7
|
+
|
8
|
+
group :test do
|
9
|
+
gem "rspec", "1.1.11"
|
10
|
+
gem "rspec-core", "2.7.1"
|
11
|
+
gem "rspec-expectations", "2.7.0"
|
12
|
+
gem "rspec-mocks", "2.7.0"
|
13
|
+
gem "guard-rspec"
|
14
|
+
end
|
data/README.md
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
ac-summarization-utils gem
|
2
|
+
==========================
|
3
|
+
|
4
|
+
### Description ###
|
5
|
+
|
6
|
+
This gem has all necessary utils to implement summarization workers:
|
7
|
+
|
8
|
+
* lookup_cache - a class that implements various lookups in the DB with REDIS as its cache
|
9
|
+
* validation_utils - utils that cehck if clicks and layer impressions are valid
|
10
|
+
* summarization_utils - utils tjat summarize records by various summarization functions (currently - sum and count are supported)
|
11
|
+
* results_dal - a class that handles insertion of summarization output to Mysql DB.
|
12
|
+
|
13
|
+
|
14
|
+
### Unit tests ###
|
15
|
+
|
16
|
+
Use the rspec framework for unit testing various utils.
|
17
|
+
|
18
|
+
$> bundle install
|
19
|
+
$> cd spec
|
20
|
+
$> rspec *.rb
|
21
|
+
|
22
|
+
|
data/Rakefile
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
$:.push File.expand_path("../lib", __FILE__)
|
3
|
+
require "ac-summarization-utils/version"
|
4
|
+
|
5
|
+
Gem::Specification.new do |s|
|
6
|
+
s.name = "ac-summarization-utils"
|
7
|
+
s.version = ACSummarizationUtils::VERSION
|
8
|
+
s.platform = Gem::Platform::RUBY
|
9
|
+
s.authors = ["Efrat Safanov"]
|
10
|
+
s.email = ["efrat@kontera.com"]
|
11
|
+
s.homepage = ""
|
12
|
+
s.summary = %q{ac-summarization-utils files}
|
13
|
+
s.description = %q{A set of common classes and utilities for the ac-summarization workers}
|
14
|
+
|
15
|
+
s.rubyforge_project = "ac-summarization-utils"
|
16
|
+
|
17
|
+
s.files = `git ls-files`.split("\n")
|
18
|
+
s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
|
19
|
+
s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
|
20
|
+
s.require_paths = ["lib"]
|
21
|
+
s.add_dependency('redis')
|
22
|
+
s.add_dependency('mysql2')
|
23
|
+
s.add_dependency('a15-worker')
|
24
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
2
|
+
<module type="RUBY_MODULE" version="4">
|
3
|
+
<component name="FacetManager">
|
4
|
+
<facet type="RailsFacetType" name="Ruby on Rails">
|
5
|
+
<configuration>
|
6
|
+
<RAILS_FACET_CONFIG_ID NAME="RAILS_FACET_SUPPORT_REMOVED" VALUE="false" />
|
7
|
+
<RAILS_FACET_CONFIG_ID NAME="RAILS_TESTS_SOURCES_PATCHED" VALUE="true" />
|
8
|
+
<RAILS_FACET_CONFIG_ID NAME="RAILS_FACET_APPLICATION_ROOT" VALUE="$MODULE_DIR$" />
|
9
|
+
</configuration>
|
10
|
+
</facet>
|
11
|
+
</component>
|
12
|
+
<component name="NewModuleRootManager" inherit-compiler-output="true">
|
13
|
+
<exclude-output />
|
14
|
+
<content url="file://$MODULE_DIR$">
|
15
|
+
<sourceFolder url="file://$MODULE_DIR$/test" isTestSource="true" />
|
16
|
+
<sourceFolder url="file://$MODULE_DIR$/spec" isTestSource="true" />
|
17
|
+
</content>
|
18
|
+
<orderEntry type="jdk" jdkName="RVM: ruby-1.9.3-p0 [summarization]" jdkType="RUBY_SDK" />
|
19
|
+
<orderEntry type="sourceFolder" forTests="false" />
|
20
|
+
<orderEntry type="library" scope="PROVIDED" name="a15-worker (v0.0.13, RVM: ruby-1.9.3-p0 [summarization]) [gem]" level="application" />
|
21
|
+
<orderEntry type="library" scope="PROVIDED" name="amq-client (v0.9.2, RVM: ruby-1.9.3-p0 [summarization]) [gem]" level="application" />
|
22
|
+
<orderEntry type="library" scope="PROVIDED" name="amq-protocol (v0.9.0, RVM: ruby-1.9.3-p0 [summarization]) [gem]" level="application" />
|
23
|
+
<orderEntry type="library" scope="PROVIDED" name="amqp (v0.9.4, RVM: ruby-1.9.3-p0 [summarization]) [gem]" level="application" />
|
24
|
+
<orderEntry type="library" scope="PROVIDED" name="bunny (v0.7.9, RVM: ruby-1.9.3-p0 [summarization]) [gem]" level="application" />
|
25
|
+
<orderEntry type="library" scope="PROVIDED" name="eventmachine (v0.12.10, RVM: ruby-1.9.3-p0 [summarization]) [gem]" level="application" />
|
26
|
+
</component>
|
27
|
+
</module>
|
28
|
+
|
@@ -0,0 +1,8 @@
|
|
1
|
+
require_relative 'ac-summarization-utils/record'
|
2
|
+
require_relative 'ac-summarization-utils/errors_handler'
|
3
|
+
require_relative 'ac-summarization-utils/summarization_utils'
|
4
|
+
require_relative 'ac-summarization-utils/lookup_cache'
|
5
|
+
require_relative 'ac-summarization-utils/results_dal'
|
6
|
+
require_relative 'ac-summarization-utils/validation_utils'
|
7
|
+
require_relative 'ac-summarization-utils/revenue_calc_utils'
|
8
|
+
require_relative 'ac-summarization-utils/general_utils'
|
@@ -0,0 +1,77 @@
|
|
1
|
+
require 'a15-worker'
|
2
|
+
|
3
|
+
module ACSummarizationUtils
|
4
|
+
|
5
|
+
class ErrorsHandler
|
6
|
+
include A15::Worker
|
7
|
+
|
8
|
+
def initialize(rabbit_db_host, exchange_input_errors, exchange_db_errors, error_key, message_type, original_rabbit_db_host=nil)
|
9
|
+
@hostname =`hostname`.strip
|
10
|
+
@error_key = error_key
|
11
|
+
@message_type = message_type
|
12
|
+
@rabbit_db_host = rabbit_db_host
|
13
|
+
input_error_queue_options = {
|
14
|
+
:host => @rabbit_db_host,
|
15
|
+
:exchange_name => exchange_input_errors
|
16
|
+
}
|
17
|
+
@input_error_producer = Producer.new(input_error_queue_options)
|
18
|
+
queries_error_queue_options = {
|
19
|
+
:host => @rabbit_db_host,
|
20
|
+
:exchange_name => exchange_db_errors
|
21
|
+
}
|
22
|
+
@queries_error_producer = Producer.new(queries_error_queue_options)
|
23
|
+
@original_rabbit_db_host = original_rabbit_db_host
|
24
|
+
end
|
25
|
+
|
26
|
+
def send_input_errors(logger, messages)
|
27
|
+
messages.each do |message|
|
28
|
+
publish_to_error_exchange(logger, @input_error_producer,message, @error_key)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def gen_and_send_input_errors(logger, messages, e)
|
33
|
+
print_error(logger, "Caught exception of type #{e.class} ,error message - #{e.message}, sending erroneous messages to a different queue")
|
34
|
+
messages.each do |message|
|
35
|
+
publish_to_error_exchange(logger, @input_error_producer,generate_error_message(message,e), @error_key)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def gen_and_send_db_errors(logger, messages, e)
|
40
|
+
print_error(logger, "Caught exception of type #{e.class} ,error message - #{e.message},queries - #{e.queries} - sending erroneous messages to a different queue")
|
41
|
+
queries_error_message = generate_error_message(messages,e)
|
42
|
+
queries_error_message[:queries] = e.queries
|
43
|
+
publish_to_error_exchange(logger, @queries_error_producer,queries_error_message, @error_key)
|
44
|
+
end
|
45
|
+
|
46
|
+
#generate an error message out of an input message
|
47
|
+
def generate_error_message(message,exception)
|
48
|
+
hostname =`hostname`.strip
|
49
|
+
rabbit_host = @original_rabbit_db_host ? @original_rabbit_db_host : @rabbit_db_host
|
50
|
+
return {:messages => message, :hostname => hostname, :key => @error_key, :rabbit_host => rabbit_host, :message_type => @message_type,
|
51
|
+
:exception=>{:message=>exception.message,:backtrace=>exception.backtrace, :class=>exception.class}}
|
52
|
+
end
|
53
|
+
|
54
|
+
def handle_fatal(logger, e)
|
55
|
+
print_error(logger, "Caught fatal exception of type #{e.class} - #{e.message}, #{e.backtrace} - re-raising exception")
|
56
|
+
raise e
|
57
|
+
end
|
58
|
+
|
59
|
+
private
|
60
|
+
def publish_to_error_exchange(logger, error_producer,message, key)
|
61
|
+
begin
|
62
|
+
error_producer.publish(Base64.encode64(Zlib::Deflate.deflate(message.to_json, 9)), {:key => key})
|
63
|
+
logger.log(:info, "Published message to error queue. Message: #{message.inspect}")
|
64
|
+
rescue Exception => e
|
65
|
+
print_error(logger, "Unable to publish to errors exchange: producer - #{error_producer} - #{e.message}, #{e.backtrace} - rejecting messages back to queue")
|
66
|
+
raise e
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
#print an error message both to logger and to stderr
|
71
|
+
def print_error(logger,err_msg_str)
|
72
|
+
logger.log(:error,err_msg_str)
|
73
|
+
$stderr.puts err_msg_str
|
74
|
+
end
|
75
|
+
|
76
|
+
end
|
77
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
#implements queries to DB with Redis cache (need to support pre-fetch)
|
2
|
+
require 'set'
|
3
|
+
|
4
|
+
module ACSummarizationUtils
|
5
|
+
module GeneralUtils
|
6
|
+
|
7
|
+
SUPPORTED_COUNTRIES = ['us','gb','it','fr','de','es','ca','in','au','ru','za'].to_set
|
8
|
+
SUPPORTED_LATIN_COUNTRIES = ['mx','bz','cr','gt','sv','hn','ni','pa','ar','bo','br','cl','co','ec','py','pe','uy','ve'].to_set
|
9
|
+
SUPPORTED_FAR_EAST = ['cn','hk','tw','sg','my','ph','jp'].to_set
|
10
|
+
|
11
|
+
def convert_country_code(country_code)
|
12
|
+
case
|
13
|
+
when SUPPORTED_LATIN_COUNTRIES.include?(country_code)
|
14
|
+
'latin_am'
|
15
|
+
when SUPPORTED_FAR_EAST.include?(country_code)
|
16
|
+
'f_east'
|
17
|
+
when SUPPORTED_COUNTRIES.include?(country_code)
|
18
|
+
country_code
|
19
|
+
else
|
20
|
+
'oth'
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
@@ -0,0 +1,480 @@
|
|
1
|
+
#implements queries to DB with Redis cache (need to support pre-fetch)
|
2
|
+
|
3
|
+
|
4
|
+
require 'mysql2'
|
5
|
+
require 'redis'
|
6
|
+
require 'time'
|
7
|
+
require 'json'
|
8
|
+
|
9
|
+
module ACSummarizationUtils
|
10
|
+
|
11
|
+
class LookupCache
|
12
|
+
|
13
|
+
ADGROUP_KEY_PREFIX = "k:adgroup"
|
14
|
+
ADGROUP_QUERY = "select adgroup_id from adc_ads ads where ads.id = %AD_ID%;"
|
15
|
+
|
16
|
+
CAMPAIGN_KEY_PREFIX = "k:campaign"
|
17
|
+
CAMPAIGN_QUERY = "select campaign_id from adc_adgroups adg join adc_ads ads on (ads.adgroup_id = adg.id) where ads.id = %AD_ID%;"
|
18
|
+
|
19
|
+
ACCOUNT_KEY_PREFIX = "k:account"
|
20
|
+
ACCOUNT_QUERY = "select account_id from adc_campaigns cmp join
|
21
|
+
adc_adgroups adg on (adg.campaign_id = cmp.id) join adc_ads ads on (ads.adgroup_id = adg.id) where ads.id = %AD_ID%;"
|
22
|
+
|
23
|
+
ADGROUP_TOPICS_KEY_PREFIX = "k:adgroup_topics"
|
24
|
+
ADGROUP_TOPICS_QUERY = "select Distinct(subTopic_id) as subtopic FROM adc_adgroups_subtopics aat JOIN adc_adgroup_settings aas
|
25
|
+
USING (adgroup_id) WHERE aat.adgroup_id = %ADGROUP_ID% AND aas.is_topical = 1"
|
26
|
+
|
27
|
+
ADGROUP_PUBLISHER_PUBGROUPS_KEY_PREFIX = "k:adgroup_publisher_pubgroups"
|
28
|
+
ADGROUP_PUBLISHER_PUBGROUPS_QUERY = "SELECT DISTINCT(PGP.publisherGroup_ID) AS pubgroup_id FROM PublisherGroupPlacement PGP JOIN adc_adgroups_pubgroups AAP ON (PGP.publisherGroup_ID = AAP.pubgroup_id) WHERE PGP.publisher_ID=%PUBLISHER_ID% AND AAP.adgroup_id=%ADGROUP_ID%"
|
29
|
+
|
30
|
+
ADVERTISER_SHARES_KEY_PREFIX = "k:advertiser_shares"
|
31
|
+
ADVERTISER_SHARES_QUERY = "SELECT advertiserShare,reserveShare FROM Advertiser WHERE advertiser_id = %ADVERTISER_ID%"
|
32
|
+
CAMPAIGN_SHARE_QUERY = "SELECT reserve/100 AS reserve FROM adc_campaign_financial_settings WHERE campaign_id = %CAMPAIGN_ID%"
|
33
|
+
|
34
|
+
PUBLISHER_PAYMENT_INFO_KEY_PREFIX = "k:publisher_payments_info"
|
35
|
+
PUBLISHER_PAYMENT_INFO_QUERY = "SELECT site_payment_type_id, rev_share, cpm FROM sites WHERE id = %PUBLISHER_ID%"
|
36
|
+
|
37
|
+
CAMPAIGN_CPM_PREFIX = "k:campaign_cpm"
|
38
|
+
CAMPAIGN_CPM_QUERY = "select CASE pricing_type_id WHEN 2 THEN pricing_rate ELSE 0 END AS cpm_rate from adc_campaign_financial_settings where campaign_id = %CAMPAIGN_ID%"
|
39
|
+
|
40
|
+
CAMPAIGN_CPC_PREFIX = "k:campaign_cpc"
|
41
|
+
CAMPAIGN_CPC_QUERY = "select CASE pricing_type_id WHEN 1 THEN pricing_rate ELSE 0 END AS cpc_rate from adc_campaign_financial_settings where campaign_id = %CAMPAIGN_ID%"
|
42
|
+
|
43
|
+
CAMPAIGN_PRICING_RATE_PREFIX = "k:campaign_pricing_rate"
|
44
|
+
CAMPAIGN_PRICING_RATE_QUERY = "select pricing_rate from adc_campaign_financial_settings where campaign_id = %CAMPAIGN_ID%"
|
45
|
+
|
46
|
+
|
47
|
+
CHANNEL_PUBLISHERS_PREFIX = "k:channel_publishers"
|
48
|
+
CHANNEL_PUBLISHERS_QUERY = "select id from sites where channel_enabled = 1;"
|
49
|
+
|
50
|
+
CAMPAIGN_PRICING_TYPE_PREFIX = "k:pricing_type_id"
|
51
|
+
CAMPAIGN_PRICING_TYPE_QUERY = "SELECT pricing_type_id FROM adc_campaign_financial_settings WHERE campaign_id = %CAMPAIGN_ID%"
|
52
|
+
|
53
|
+
ENGAGEMENT_AD_TYPE_PREFIX = "k:engagement_ad_type"
|
54
|
+
ENGAGEMENT_AD_TYPE_QUERY = "SELECT id, ad_type_id FROM adc_ads WHERE id = %AD_ID%"
|
55
|
+
|
56
|
+
ENGAGEMENT_AD_VIDEO_PREFIX = "k:engagement_ad_video"
|
57
|
+
ENGAGEMENT_AD_VIDEO_QUERY = "SELECT ad_id, ROUND(IF(rm_time IS NULL, 0, rm_time*1000)) as rm_time_millis FROM adc_ad_settings WHERE ad_id = %AD_ID%"
|
58
|
+
|
59
|
+
ENGAGEMENT_THRESHOLD_PREFIX = "k:engagement_threshold"
|
60
|
+
ENGAGEMENT_THRESHOLD_QUERY = "SELECT advanced_setting_value as engagement_qualifier FROM adc_campaign_advanced_settings WHERE campaign_id = %CAMPAIGN_ID% AND advanced_setting_type_id = 6"
|
61
|
+
|
62
|
+
ENGAGEMENT_BY_PERCENTAGE_PREFIX = "k:engagment_by_percentage"
|
63
|
+
ENGAGEMENT_BY_PERCENTAGE_QUERY = "SELECT IF(engagement_report_by_percentage IS NULL, 0, engagement_report_by_percentage) as engagment_by_percentage FROM adc_ad_settings WHERE ad_id = %AD_ID%"
|
64
|
+
|
65
|
+
INIMAGE_KONTERA_PUBLISHER_ID_PREFIX = "k:inimage_publisher_id_map"
|
66
|
+
INIMAGE_KONTERA_PUBLISHER_ID_QUERY = "SELECT publisher_id FROM js_publisherparams LEFT JOIN js_params JP on js_param_id=JP.id where JP.name=\"luminate_account_id\" and paramValue = \"%INIMAGE_PUBLISHER_ID%\""
|
67
|
+
|
68
|
+
INIMAGE_PAYMENT_TYPE_PREFIX = "k:inimage_payment_type_id"
|
69
|
+
INIMAGE_PAYMENT_TYPE_QUERY = "select IFNULL(paramValue,defaultValue) as payment_type_id from (select A.name, A.defaultValue, B.publisher_id, B.paramValue from js_params as A left join (select * from js_publisherparams where publisher_id=\"%PUBLISHER_ID%\") as B on A.id=B.js_param_id where name=\"inimage_payment_type_id\") as C"
|
70
|
+
|
71
|
+
INIMAGE_CPM_VALUE_PREFIX = "k:inimage_cpm_value"
|
72
|
+
INIMAGE_CPM_VALUE_QUERY = "select IFNULL(paramValue,defaultValue) as cpm from (select A.name, A.defaultValue, B.publisher_id, B.paramValue from js_params as A left join (select * from js_publisherparams where publisher_id=\"%PUBLISHER_ID%\") as B on A.id=B.js_param_id where name=\"inimage_cpm\") as C"
|
73
|
+
|
74
|
+
INIMAGE_REV_SHARE_PREFIX = "k:inimage_rev_share"
|
75
|
+
INIMAGE_REV_SHARE_QUERY = "select IFNULL(paramValue,defaultValue) as rev_share from (select A.name, A.defaultValue, B.publisher_id, B.paramValue from js_params as A left join (select * from js_publisherparams where publisher_id=\"%PUBLISHER_ID%\") as B on A.id=B.js_param_id where name=\"inimage_rev_share\") as C"
|
76
|
+
|
77
|
+
INIMAGE_PUBLISHER_PAYMENT_INFO_KEY_PREFIX = "k:inimage_publisher_payments_info"
|
78
|
+
|
79
|
+
DUP_KEY_PREFIX = "k:dup"
|
80
|
+
|
81
|
+
class ResourcesError < RuntimeError; end
|
82
|
+
class InputError < RuntimeError; end
|
83
|
+
|
84
|
+
#throws exception if there is a resources connection problem
|
85
|
+
def initialize(redis_host, redis_port, redis_expiration, db_host, db_user, db_password, db_name)
|
86
|
+
@cache_expiration = redis_expiration.to_i
|
87
|
+
begin
|
88
|
+
@cache_client = Redis.connect({:host => redis_host, :port => redis_port.to_i })
|
89
|
+
@mysql_client = Mysql2::Client.new({:host => db_host, :username => db_user, :password => db_password, :database => db_name, :reconnect => true})
|
90
|
+
rescue Exception => e
|
91
|
+
@cache_client.client.disconnect if @cache_client and @cache_client.client and @cache_client.client.connected?
|
92
|
+
raise ResourcesError, e.message
|
93
|
+
end
|
94
|
+
end
|
95
|
+
def self.convert_url_to_host_and_port(options)
|
96
|
+
require "uri"
|
97
|
+
return options if !options[:url]
|
98
|
+
url = URI(options.delete(:url))
|
99
|
+
options[:host] = url.host
|
100
|
+
options[:port] = url.port
|
101
|
+
options
|
102
|
+
end
|
103
|
+
def self.connect(redis_options, lookup_db_options)
|
104
|
+
redis_options = convert_url_to_host_and_port(redis_options.dup)
|
105
|
+
new(redis_options[:host],redis_options[:port], redis_options[:expiration], lookup_db_options[:host], lookup_db_options[:user], lookup_db_options[:password], lookup_db_options[:name])
|
106
|
+
end
|
107
|
+
|
108
|
+
def get_adgroup_id(ad_id)
|
109
|
+
return get_field_by_ad_id(ad_id, ADGROUP_KEY_PREFIX, ADGROUP_QUERY, "adgroup_id")
|
110
|
+
end
|
111
|
+
|
112
|
+
def get_campaign_id(ad_id)
|
113
|
+
return get_field_by_ad_id(ad_id, CAMPAIGN_KEY_PREFIX, CAMPAIGN_QUERY, "campaign_id")
|
114
|
+
end
|
115
|
+
|
116
|
+
def get_campaign_pricing_type_id(campaign_id)
|
117
|
+
return get_field_by_campaign_id(campaign_id, CAMPAIGN_PRICING_TYPE_PREFIX, CAMPAIGN_PRICING_TYPE_QUERY, "pricing_type_id")
|
118
|
+
end
|
119
|
+
|
120
|
+
def get_account_id(ad_id)
|
121
|
+
return get_field_by_ad_id(ad_id, ACCOUNT_KEY_PREFIX, ACCOUNT_QUERY, "account_id")
|
122
|
+
end
|
123
|
+
|
124
|
+
def get_engagement_type(ad_id)
|
125
|
+
return get_field_by_ad_id(ad_id, ENGAGEMENT_AD_TYPE_PREFIX, ENGAGEMENT_AD_TYPE_QUERY, "ad_type_id")
|
126
|
+
end
|
127
|
+
|
128
|
+
def get_engagement_video_length(ad_id)
|
129
|
+
return get_field_by_ad_id(ad_id, ENGAGEMENT_AD_VIDEO_PREFIX, ENGAGEMENT_AD_VIDEO_QUERY, "rm_time_millis")
|
130
|
+
end
|
131
|
+
|
132
|
+
def get_is_engagement_by_percentage(ad_id)
|
133
|
+
return get_field_by_ad_id(ad_id, ENGAGEMENT_BY_PERCENTAGE_PREFIX, ENGAGEMENT_BY_PERCENTAGE_QUERY, "engagment_by_percentage")
|
134
|
+
end
|
135
|
+
|
136
|
+
def get_publisher_id_by_inimage_id(inimage_publisher_id)
|
137
|
+
return get_publisher_id_by_inimage_id_helper(inimage_publisher_id, INIMAGE_KONTERA_PUBLISHER_ID_PREFIX, INIMAGE_KONTERA_PUBLISHER_ID_QUERY, "publisher_id")
|
138
|
+
end
|
139
|
+
|
140
|
+
def get_engagement_threshold(campaign_id)
|
141
|
+
begin
|
142
|
+
result = get_field_by_campaign_id(campaign_id, ENGAGEMENT_THRESHOLD_PREFIX, ENGAGEMENT_THRESHOLD_QUERY, "engagement_qualifier")
|
143
|
+
return JSON.parse(result)["value"].to_i if result
|
144
|
+
rescue => e
|
145
|
+
end
|
146
|
+
return nil
|
147
|
+
end
|
148
|
+
|
149
|
+
#deprecated: moved to summarization-utils
|
150
|
+
#raises an exception if cannot parse date
|
151
|
+
def get_date(date)
|
152
|
+
return nil if date.nil?
|
153
|
+
begin
|
154
|
+
time_object = date.is_a?(Fixnum) ? Time.at(date) : Time.parse(date)
|
155
|
+
return time_object.strftime("%Y-%m-%d")
|
156
|
+
rescue Exception => e
|
157
|
+
raise InputError, "Unable to parse time string: #{date} - message: #{e.message}"
|
158
|
+
end
|
159
|
+
end
|
160
|
+
|
161
|
+
def get_relevant_topic_ids(page_topic_ids, adgroup_id)
|
162
|
+
return [] if page_topic_ids.nil? or adgroup_id.nil?
|
163
|
+
topics_array = page_topic_ids.is_a?(Array) ? page_topic_ids : page_topic_ids.split(/(\+| )/)
|
164
|
+
return [] if topics_array.nil? or topics_array.empty?
|
165
|
+
adgroup_topics_ids = get_topic_ids(adgroup_id)
|
166
|
+
return [] if adgroup_topics_ids.nil? or adgroup_topics_ids.empty?
|
167
|
+
topics_array.map! { |t| t.to_i}
|
168
|
+
adgroup_topics_ids.map! {|t| t.to_i}
|
169
|
+
return topics_array & adgroup_topics_ids
|
170
|
+
end
|
171
|
+
|
172
|
+
def get_pubgroup_ids(adgroup_id,publisher_id)
|
173
|
+
return [] if publisher_id.nil? or adgroup_id.nil?
|
174
|
+
adgroup_publisher_pubgroup_ids = get_adgroup_publisher_pubgroup_ids(adgroup_id,publisher_id)
|
175
|
+
return [] if adgroup_publisher_pubgroup_ids.nil? or adgroup_publisher_pubgroup_ids.empty?
|
176
|
+
return adgroup_publisher_pubgroup_ids.map{|pg| pg.to_i}
|
177
|
+
end
|
178
|
+
|
179
|
+
def get_and_set_duplicate_value(key,expiry_secs)
|
180
|
+
res = lookup_in_cache("#{DUP_KEY_PREFIX}:#{key}")
|
181
|
+
return res if res
|
182
|
+
add_to_cache("#{DUP_KEY_PREFIX}:#{key}",true, expiry_secs)
|
183
|
+
return nil
|
184
|
+
end
|
185
|
+
|
186
|
+
def clear_duplicate_value(key)
|
187
|
+
delete_from_cache("#{DUP_KEY_PREFIX}:#{key}")
|
188
|
+
end
|
189
|
+
|
190
|
+
def get_advertiser_shares(advertiser_id,campaign_id=nil)
|
191
|
+
return {} if advertiser_id.nil?
|
192
|
+
advertiser_shares = get_advertisers_shares_info(advertiser_id,campaign_id)
|
193
|
+
return advertiser_shares
|
194
|
+
end
|
195
|
+
|
196
|
+
def get_publisher_payment_info(publisher_id)
|
197
|
+
return {} if publisher_id.nil?
|
198
|
+
publisher_payment_info = get_publisher_payment_data(publisher_id)
|
199
|
+
return publisher_payment_info
|
200
|
+
end
|
201
|
+
|
202
|
+
def get_inimage_publisher_payment_info(publisher_id)
|
203
|
+
return {} if publisher_id.nil?
|
204
|
+
publisher_payment_info = get_inimage_publisher_payment_data(publisher_id)
|
205
|
+
return publisher_payment_info
|
206
|
+
end
|
207
|
+
|
208
|
+
def get_campaign_cpm(campaign_id)
|
209
|
+
return campaign_id.nil? ? 0.0 : get_field_by_campaign_id(campaign_id, CAMPAIGN_CPM_PREFIX, CAMPAIGN_CPM_QUERY, "cpm_rate")
|
210
|
+
end
|
211
|
+
|
212
|
+
def get_campaign_cpc(campaign_id)
|
213
|
+
return campaign_id.nil? ? 0.0 : get_field_by_campaign_id(campaign_id, CAMPAIGN_CPC_PREFIX, CAMPAIGN_CPC_QUERY, "cpc_rate")
|
214
|
+
end
|
215
|
+
|
216
|
+
def get_campaign_pricing_rate(campaign_id)
|
217
|
+
get_field_by_campaign_id(campaign_id, CAMPAIGN_PRICING_RATE_PREFIX, CAMPAIGN_PRICING_RATE_QUERY, "pricing_rate")
|
218
|
+
end
|
219
|
+
|
220
|
+
def get_channel_publishers()
|
221
|
+
return get_channel_publishers_data()
|
222
|
+
end
|
223
|
+
|
224
|
+
private
|
225
|
+
|
226
|
+
def get_topic_ids(adgroup_id)
|
227
|
+
cache_result = get_adgroup_topics_from_cache(adgroup_id)
|
228
|
+
return cache_result if cache_result
|
229
|
+
db_result = get_adgroup_topics_from_db(adgroup_id)
|
230
|
+
add_to_cache("#{ADGROUP_TOPICS_KEY_PREFIX}:#{adgroup_id}", db_result.to_json)
|
231
|
+
return db_result
|
232
|
+
end
|
233
|
+
|
234
|
+
def get_adgroup_publisher_pubgroup_ids(adgroup_id,publisher_id)
|
235
|
+
cache_result = get_adgroup_publisher_pubgroups_from_cache(adgroup_id,publisher_id)
|
236
|
+
return cache_result if cache_result
|
237
|
+
db_result = get_adgroup_publisher_pubgroups_from_db(adgroup_id,publisher_id)
|
238
|
+
add_to_cache("#{ADGROUP_PUBLISHER_PUBGROUPS_KEY_PREFIX}:#{adgroup_id}:#{publisher_id}",db_result.to_json)
|
239
|
+
return db_result
|
240
|
+
end
|
241
|
+
|
242
|
+
def get_advertisers_shares_info(advertiser_id,campaign_id=nil)
|
243
|
+
key_suffix = campaign_id.nil? ? ":#{advertiser_id}" : ":#{advertiser_id}:#{campaign_id}"
|
244
|
+
cache_result = get_advertiser_shares_from_cache(key_suffix)
|
245
|
+
return cache_result if cache_result
|
246
|
+
db_result = get_advertiser_shares_from_db(advertiser_id,campaign_id)
|
247
|
+
add_to_cache("#{ADVERTISER_SHARES_KEY_PREFIX}#{key_suffix}",db_result.to_json)
|
248
|
+
return db_result
|
249
|
+
end
|
250
|
+
|
251
|
+
def get_publisher_payment_data(publisher_id)
|
252
|
+
cache_result = get_publisher_payment_info_from_cache(publisher_id)
|
253
|
+
return cache_result if cache_result
|
254
|
+
db_result = get_publisher_payment_info_from_db(publisher_id)
|
255
|
+
add_to_cache("#{PUBLISHER_PAYMENT_INFO_KEY_PREFIX}:#{publisher_id}",db_result.to_json)
|
256
|
+
return db_result
|
257
|
+
end
|
258
|
+
|
259
|
+
def get_field_by_ad_id(ad_id, cache_key_prefix, db_query, db_field)
|
260
|
+
return nil if ad_id.nil?
|
261
|
+
cache_key = "#{cache_key_prefix}:#{ad_id}"
|
262
|
+
cache_result = lookup_in_cache(cache_key)
|
263
|
+
return cache_result if cache_result
|
264
|
+
db_result = lookup_in_db(db_query, ad_id, db_field)
|
265
|
+
if db_result
|
266
|
+
add_to_cache(cache_key, db_result)
|
267
|
+
return db_result
|
268
|
+
else
|
269
|
+
raise InputError, "Can't get #{db_field} from DB for ad_id: #{ad_id}"
|
270
|
+
end
|
271
|
+
end
|
272
|
+
|
273
|
+
def get_adgroup_topics_from_cache(adgroup_id)
|
274
|
+
res = lookup_in_cache("#{ADGROUP_TOPICS_KEY_PREFIX}:#{adgroup_id}")
|
275
|
+
return res.nil? ? nil : JSON.parse(res)
|
276
|
+
end
|
277
|
+
|
278
|
+
def get_adgroup_topics_from_db(adgroup_id)
|
279
|
+
query = ADGROUP_TOPICS_QUERY.gsub("%ADGROUP_ID%", "#{adgroup_id}")
|
280
|
+
topic_ids = []
|
281
|
+
begin
|
282
|
+
result = @mysql_client.query(query)
|
283
|
+
return [] if result.nil? or result.size == 0
|
284
|
+
result.each do |row|
|
285
|
+
topic_ids << row["subtopic"] unless row["subtopic"].nil?
|
286
|
+
end
|
287
|
+
rescue Mysql2::Error => e
|
288
|
+
raise ResourcesError, "Got MysqL exception in lookup for query - '#{query}'. Message: #{e.message}"
|
289
|
+
end
|
290
|
+
return topic_ids
|
291
|
+
end
|
292
|
+
|
293
|
+
def get_adgroup_publisher_pubgroups_from_cache(adgroup_id,publisher_id)
|
294
|
+
res = lookup_in_cache("#{ADGROUP_PUBLISHER_PUBGROUPS_KEY_PREFIX}:#{adgroup_id}:#{publisher_id}")
|
295
|
+
return res.nil? ? nil : JSON.parse(res)
|
296
|
+
end
|
297
|
+
|
298
|
+
def get_adgroup_publisher_pubgroups_from_db(adgroup_id,publisher_id)
|
299
|
+
query = ADGROUP_PUBLISHER_PUBGROUPS_QUERY.gsub("%ADGROUP_ID%", "#{adgroup_id}").gsub("%PUBLISHER_ID%","#{publisher_id}")
|
300
|
+
pubgroup_ids = []
|
301
|
+
begin
|
302
|
+
result = @mysql_client.query(query)
|
303
|
+
return [-1] if result.nil? or result.size == 0
|
304
|
+
result.each do |row|
|
305
|
+
pubgroup_ids << row["pubgroup_id"] unless row["pubgroup_id"].nil?
|
306
|
+
end
|
307
|
+
rescue Mysql2::Error => e
|
308
|
+
raise ResourcesError, "Got MysqL exception in lookup for query - '#{query}'. Message: #{e.message}"
|
309
|
+
end
|
310
|
+
return pubgroup_ids
|
311
|
+
end
|
312
|
+
|
313
|
+
def get_advertiser_shares_from_cache(key_suffix)
|
314
|
+
res = lookup_in_cache("#{ADVERTISER_SHARES_KEY_PREFIX}#{key_suffix}")
|
315
|
+
return res.nil? ? nil : JSON.parse(res)
|
316
|
+
end
|
317
|
+
|
318
|
+
def get_advertiser_shares_from_db(advertiser_id,campaign_id=nil)
|
319
|
+
camp_query = CAMPAIGN_SHARE_QUERY.gsub("%CAMPAIGN_ID%","#{campaign_id}") if !campaign_id.nil?
|
320
|
+
shares = {"reserve_share" => 0.0, "adv_share" => 0.0, "campaign_reserve" => 0.0}
|
321
|
+
begin
|
322
|
+
adv_result = @mysql_client.query(ADVERTISER_SHARES_QUERY.gsub("%ADVERTISER_ID%","#{advertiser_id}"))
|
323
|
+
camp_result = @mysql_client.query(camp_query) if !campaign_id.nil?
|
324
|
+
return shares if ((adv_result.nil? or adv_result.size == 0) and (camp_result.nil? or camp_result.size == 0))
|
325
|
+
if (!adv_result.first.nil?)
|
326
|
+
shares["adv_share"] = adv_result.first["advertiserShare"].to_f unless adv_result.first["advertiserShare"].nil?
|
327
|
+
shares["reserve_share"] = adv_result.first["reserveShare"].to_f unless adv_result.first["reserveShare"].nil?
|
328
|
+
end
|
329
|
+
shares["campaign_reserve"] = camp_result.first["reserve"].to_f unless (camp_result.nil? || camp_result.first.nil? || camp_result.first["reserve"].nil?)
|
330
|
+
end
|
331
|
+
|
332
|
+
return shares
|
333
|
+
end
|
334
|
+
|
335
|
+
def get_publisher_payment_info_from_cache(publisher_id)
|
336
|
+
res = lookup_in_cache("#{PUBLISHER_PAYMENT_INFO_KEY_PREFIX}:#{publisher_id}")
|
337
|
+
return res.nil? ? nil : JSON.parse(res)
|
338
|
+
end
|
339
|
+
|
340
|
+
def get_publisher_payment_info_from_db(publisher_id)
|
341
|
+
publisher_payment_info = {"payment_type" => 1, "cpm" => 0.0, "rev_share" => 0.0}
|
342
|
+
res = @mysql_client.query(PUBLISHER_PAYMENT_INFO_QUERY.gsub("%PUBLISHER_ID%","#{publisher_id}"))
|
343
|
+
return publisher_payment_info if (res.nil? or res.size == 0)
|
344
|
+
publisher_payment_info["payment_type"] = res.first["site_payment_type_id"]
|
345
|
+
publisher_payment_info["rev_share"] = res.first["rev_share"].to_f
|
346
|
+
publisher_payment_info["cpm"] = res.first["cpm"].to_f
|
347
|
+
return publisher_payment_info
|
348
|
+
end
|
349
|
+
|
350
|
+
def get_field_by_campaign_id(campaign_id, cache_key_prefix, db_query, db_field)
|
351
|
+
cache_key = "#{cache_key_prefix}:#{campaign_id}"
|
352
|
+
cache_result = lookup_in_cache(cache_key)
|
353
|
+
return cache_result if cache_result
|
354
|
+
db_result = get_field_by_campaign_from_db(db_query, campaign_id, db_field)
|
355
|
+
if db_result
|
356
|
+
add_to_cache(cache_key, db_result)
|
357
|
+
return db_result
|
358
|
+
else
|
359
|
+
raise InputError, "Can't get #{db_field} from DB for campaign_id: #{campaign_id}"
|
360
|
+
end
|
361
|
+
end
|
362
|
+
|
363
|
+
def get_channel_publishers_data()
|
364
|
+
cache_key = "#{CHANNEL_PUBLISHERS_PREFIX}"
|
365
|
+
cache_result = lookup_in_cache(cache_key)
|
366
|
+
return cache_result if cache_result
|
367
|
+
channel_publishers = []
|
368
|
+
begin
|
369
|
+
result = @mysql_client.query(CHANNEL_PUBLISHERS_QUERY)
|
370
|
+
return [] if result.nil? or result.size == 0
|
371
|
+
result.each do |row|
|
372
|
+
channel_publishers << row["id"] unless row["id"].nil?
|
373
|
+
end
|
374
|
+
rescue Mysql2::Error => e
|
375
|
+
raise ResourcesError, "Got MysqL exception in lookup for query - '#{CHANNEL_PUBLISHERS_QUERY}'. Message: #{e.message}"
|
376
|
+
end
|
377
|
+
add_to_cache(cache_key, channel_publishers )
|
378
|
+
return channel_publishers
|
379
|
+
end
|
380
|
+
|
381
|
+
def add_to_cache(key, value, expiry = @cache_expiration)
|
382
|
+
begin
|
383
|
+
@cache_client.setex(key, expiry, value)
|
384
|
+
rescue Exception => e
|
385
|
+
raise ResourcesError, "Got Redis exception in adding to cache for key - '#{key}' with value: '#{value}'. Message: #{e.message}"
|
386
|
+
end
|
387
|
+
end
|
388
|
+
|
389
|
+
def delete_from_cache(key)
|
390
|
+
begin
|
391
|
+
@cache_client.del(key)
|
392
|
+
rescue Exception => e
|
393
|
+
raise ResourcesError, "Got Redis exception in deleting from cache - '#{key}'. Message: #{e.message}"
|
394
|
+
end
|
395
|
+
end
|
396
|
+
|
397
|
+
def lookup_in_cache(key)
|
398
|
+
begin
|
399
|
+
return @cache_client.get(key)
|
400
|
+
rescue Exception => e
|
401
|
+
raise ResourcesError, "Got Redis exception in getting key - '#{key}'. Message: #{e.message}"
|
402
|
+
end
|
403
|
+
end
|
404
|
+
|
405
|
+
def lookup_in_db(query, val, res_field_name)
|
406
|
+
full_query = query.gsub("%AD_ID%", "#{val}")
|
407
|
+
begin
|
408
|
+
result = @mysql_client.query(full_query)
|
409
|
+
return nil if result.nil? or result.size == 0
|
410
|
+
return result.first[res_field_name]
|
411
|
+
rescue Mysql2::Error => e
|
412
|
+
raise ResourcesError, "Got MysqL exception in lookup for query - '#{full_query}'. Message: #{e.message}"
|
413
|
+
end
|
414
|
+
end
|
415
|
+
|
416
|
+
def get_field_by_campaign_from_db(query, campaign_id, res_field_name)
|
417
|
+
full_query = query.gsub("%CAMPAIGN_ID%", "#{campaign_id}")
|
418
|
+
begin
|
419
|
+
result = @mysql_client.query(full_query)
|
420
|
+
return nil if result.nil? or result.size == 0
|
421
|
+
return result.first[res_field_name]
|
422
|
+
rescue Mysql2::Error => e
|
423
|
+
raise ResourcesError, "Got MysqL exception in lookup for query - '#{full_query}'. Message: #{e.message}"
|
424
|
+
end
|
425
|
+
end
|
426
|
+
|
427
|
+
def get_publisher_id_by_inimage_id_from_db(query, inimage_publisher_id, res_field_name)
|
428
|
+
full_query = query.gsub("%INIMAGE_PUBLISHER_ID%", "#{inimage_publisher_id}")
|
429
|
+
begin
|
430
|
+
result = @mysql_client.query(full_query)
|
431
|
+
return nil if result.nil? or result.size == 0
|
432
|
+
return result.first[res_field_name]
|
433
|
+
rescue Mysql2::Error => e
|
434
|
+
raise ResourcesError, "Got MysqL exception in lookup for query - '#{full_query}'. Message: #{e.message}"
|
435
|
+
end
|
436
|
+
end
|
437
|
+
|
438
|
+
def get_publisher_id_by_inimage_id_helper(inimage_publisher_id, cache_key_prefix, db_query, db_field)
|
439
|
+
return nil if inimage_publisher_id.nil?
|
440
|
+
cache_key = "#{cache_key_prefix}:#{inimage_publisher_id}"
|
441
|
+
cache_result = lookup_in_cache(cache_key)
|
442
|
+
return cache_result if cache_result
|
443
|
+
db_result = get_publisher_id_by_inimage_id_from_db(db_query, inimage_publisher_id, db_field)
|
444
|
+
if db_result
|
445
|
+
add_to_cache(cache_key, db_result)
|
446
|
+
return db_result
|
447
|
+
else
|
448
|
+
raise InputError, "Can't get #{db_field} from DB for inimage_publisher_id: #{inimage_publisher_id}"
|
449
|
+
end
|
450
|
+
end
|
451
|
+
|
452
|
+
def get_inimage_publisher_payment_info_from_cache(publisher_id)
|
453
|
+
res = lookup_in_cache("#{INIMAGE_PUBLISHER_PAYMENT_INFO_KEY_PREFIX}:#{publisher_id}")
|
454
|
+
return res.nil? ? nil : JSON.parse(res)
|
455
|
+
end
|
456
|
+
|
457
|
+
def get_inimage_publisher_payment_info_from_db(publisher_id)
|
458
|
+
publisher_payment_info = {"payment_type" => 1, "cpm" => 0.0, "rev_share" => 0.0}
|
459
|
+
res_payment_type_id = @mysql_client.query(INIMAGE_PAYMENT_TYPE_QUERY.gsub("%PUBLISHER_ID%","#{publisher_id}"))
|
460
|
+
res_cpm_value = @mysql_client.query(INIMAGE_CPM_VALUE_QUERY.gsub("%PUBLISHER_ID%","#{publisher_id}"))
|
461
|
+
res_rev_share = @mysql_client.query(INIMAGE_REV_SHARE_QUERY.gsub("%PUBLISHER_ID%","#{publisher_id}"))
|
462
|
+
return publisher_payment_info if (res_payment_type_id.nil? or res_payment_type_id.size == 0)
|
463
|
+
publisher_payment_info["payment_type"] = res_payment_type_id.first["payment_type_id"].to_i
|
464
|
+
publisher_payment_info["rev_share"] = (res_rev_share.nil? or res_rev_share.size == 0) ? 0.0 : res_rev_share.first["rev_share"].to_f
|
465
|
+
publisher_payment_info["cpm"] = (res_cpm_value.nil? or res_cpm_value.size == 0) ? 0.0 : res_cpm_value.first["cpm"].to_f
|
466
|
+
|
467
|
+
return publisher_payment_info
|
468
|
+
end
|
469
|
+
|
470
|
+
def get_inimage_publisher_payment_data(publisher_id)
|
471
|
+
cache_result = get_inimage_publisher_payment_info_from_cache(publisher_id)
|
472
|
+
return cache_result if cache_result
|
473
|
+
db_result = get_inimage_publisher_payment_info_from_db(publisher_id)
|
474
|
+
add_to_cache("#{INIMAGE_PUBLISHER_PAYMENT_INFO_KEY_PREFIX}:#{publisher_id}",db_result.to_json)
|
475
|
+
return db_result
|
476
|
+
end
|
477
|
+
|
478
|
+
end
|
479
|
+
|
480
|
+
end
|