ac-summarization-utils 0.0.47.rc1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +4 -0
- data/Gemfile +14 -0
- data/README.md +22 -0
- data/Rakefile +2 -0
- data/ac-summarization-utils.gemspec +24 -0
- data/ac-summarization-utils.iml +28 -0
- data/lib/ac-summarization-utils.rb +8 -0
- data/lib/ac-summarization-utils/errors_handler.rb +77 -0
- data/lib/ac-summarization-utils/general_utils.rb +25 -0
- data/lib/ac-summarization-utils/lookup_cache.rb +480 -0
- data/lib/ac-summarization-utils/record.rb +40 -0
- data/lib/ac-summarization-utils/results_dal.rb +189 -0
- data/lib/ac-summarization-utils/revenue_calc_utils.rb +41 -0
- data/lib/ac-summarization-utils/summarization_utils.rb +119 -0
- data/lib/ac-summarization-utils/validation_utils.rb +93 -0
- data/lib/ac-summarization-utils/version.rb +3 -0
- data/spec/errors_handler_spec.rb +52 -0
- data/spec/general_utils_spec.rb +35 -0
- data/spec/lookup_cache_spec.rb +956 -0
- data/spec/results_dal_spec.rb +349 -0
- data/spec/revenue_calc_utils_spec.rb +125 -0
- data/spec/summarization_utils_spec.rb +210 -0
- data/spec/validation_utils_spec.rb +328 -0
- metadata +116 -0
@@ -0,0 +1,349 @@
|
|
1
|
+
require 'rspec'
|
2
|
+
require 'mysql2'
|
3
|
+
require_relative '../lib/ac-summarization-utils/results_dal'
|
4
|
+
|
5
|
+
include ACSummarizationUtils
|
6
|
+
|
7
|
+
describe "ResultsDAL#" do
|
8
|
+
|
9
|
+
def mock_results_db
|
10
|
+
@db_mock = mock("Mysql2::Client")
|
11
|
+
@db_mock.stub!(:escape) { |arg| arg }
|
12
|
+
Mysql2::Client.stub(:new).with(any_args()).and_return(@db_mock)
|
13
|
+
end
|
14
|
+
|
15
|
+
it "should not execute queries if no records are sent" do
|
16
|
+
mock_results_db
|
17
|
+
@db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
|
18
|
+
|
19
|
+
target = ResultsDAL.new({}, "table_name", nil)
|
20
|
+
|
21
|
+
target.insert_rows([], [], [])
|
22
|
+
target.insert_rows(nil, [], [])
|
23
|
+
end
|
24
|
+
|
25
|
+
it "should not execute queries if no columns are sent" do
|
26
|
+
mock_results_db
|
27
|
+
@db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
|
28
|
+
rows = [{"ad_id"=>"42173", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2}]
|
29
|
+
target = ResultsDAL.new({}, "table_name", nil)
|
30
|
+
|
31
|
+
target.insert_rows(rows, nil, ["clicks"])
|
32
|
+
target.insert_rows(rows, [], ["clicks"])
|
33
|
+
end
|
34
|
+
|
35
|
+
it "should execute unique queries if no duplicate fields are sent" do
|
36
|
+
mock_results_db
|
37
|
+
@db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
|
38
|
+
rows = [{"ad_id"=>"42173", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2}]
|
39
|
+
@db_mock.should_receive(:query).with("insert into table_name (id,ad_id) values \n (NULL,'42173');").exactly(2).times
|
40
|
+
target = ResultsDAL.new({}, "table_name", nil)
|
41
|
+
|
42
|
+
target.insert_rows(rows, ["ad_id"], nil)
|
43
|
+
target.insert_rows(rows, ["ad_id"], [])
|
44
|
+
end
|
45
|
+
|
46
|
+
it "should add partition suffixes to table name if they are defined (unique insert)" do
|
47
|
+
mock_results_db
|
48
|
+
@db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
|
49
|
+
rows = [{"ad_id"=>"42173", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "hour" => 12, "day" => 27, "month" => 9}]
|
50
|
+
@db_mock.should_receive(:query).with("insert into table_name_27_9_12 (id,ad_id) values \n (NULL,'42173');").exactly(2).times
|
51
|
+
target = ResultsDAL.new({}, "table_name", {:use_partitions => true, :field_name => "hour", :num_partitions => 24, :partition_prefixes => ["day", "month"]})
|
52
|
+
|
53
|
+
target.insert_rows(rows, ["ad_id"], nil)
|
54
|
+
target.insert_rows(rows, ["ad_id"], [])
|
55
|
+
end
|
56
|
+
|
57
|
+
it "should add partition suffixes to table name if they are defined (duplicate insert)" do
|
58
|
+
mock_results_db
|
59
|
+
@db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
|
60
|
+
rows = [{"ad_id"=>"42173", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "hour" => 12, "day" => 27, "month" => 9, "clicks" => 2}]
|
61
|
+
@db_mock.should_receive(:query).with("insert into table_name_27_9_12 (id,ad_id) values \n (NULL,'42173') on duplicate key update clicks = clicks + values(clicks);")
|
62
|
+
|
63
|
+
target = ResultsDAL.new({}, "table_name", {:use_partitions => true, :field_name => "hour", :num_partitions => 24, :partition_prefixes => ["day", "month"]})
|
64
|
+
|
65
|
+
target.insert_rows(rows, ["ad_id"], ["clicks"])
|
66
|
+
end
|
67
|
+
|
68
|
+
it "should raise exception if table name doesn't exist" do
|
69
|
+
mock_results_db
|
70
|
+
@db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([])
|
71
|
+
|
72
|
+
lambda{ResultsDAL.new({}, "table_name", nil)}.should raise_error(ResultsDAL::DBFatalError)
|
73
|
+
|
74
|
+
end
|
75
|
+
|
76
|
+
it "should create tables if so specified" do
|
77
|
+
mock_results_db
|
78
|
+
@db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([])
|
79
|
+
create_query = "( `id` int(11) NOT NULL auto_increment, `adgroup_id` int(11) default NULL, PRIMARY KEY (`id`),) ENGINE=InnoDB AUTO_INCREMENT=39957 DEFAULT CHARSET=utf8"
|
80
|
+
|
81
|
+
rows = [{"ad_id"=>"42173", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "hour" => 12, "day" => 27, "month" => 9, "clicks" => 2},
|
82
|
+
{"ad_id"=>"42172", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "hour" => 12, "day" => 27, "month" => 9, "clicks" => 1}]
|
83
|
+
@db_mock.should_receive(:query).with("CREATE TABLE IF NOT EXISTS table_name_27_9_12 "+create_query).and_return([])
|
84
|
+
|
85
|
+
@db_mock.should_receive(:query).with("insert into table_name_27_9_12 (id,ad_id) values \n (NULL,'42173') on duplicate key update clicks = clicks + values(clicks);")
|
86
|
+
@db_mock.should_receive(:query).with("insert into table_name_27_9_12 (id,ad_id) values \n (NULL,'42172') on duplicate key update clicks = clicks + values(clicks);")
|
87
|
+
|
88
|
+
target = ResultsDAL.new({}, "table_name", {:use_partitions => true, :field_name => "hour", :num_partitions => 24, :partition_prefixes => ["day", "month"], :create_tables => true, :create_table_columns => create_query})
|
89
|
+
target.insert_rows(rows, ["ad_id"], ["clicks"])
|
90
|
+
end
|
91
|
+
|
92
|
+
it "should raise exception if manual partitioning requested and relevant info missing" do
|
93
|
+
mock_results_db
|
94
|
+
|
95
|
+
lambda{ResultsDAL.new({}, "table_name", {:use_partitions => true})}.should raise_error(ResultsDAL::DBFatalError)
|
96
|
+
lambda{ResultsDAL.new({}, "table_name", {:use_partitions => true, :field_name => "adgroup_id"})}.should raise_error(ResultsDAL::DBFatalError)
|
97
|
+
lambda{ResultsDAL.new({}, "table_name", {:use_partitions => true, :num_partitions => 2})}.should raise_error(ResultsDAL::DBFatalError)
|
98
|
+
end
|
99
|
+
|
100
|
+
it "should raise exception if manual partitioning requested and partitions number is larger than 100 or lower/equal than 0" do
|
101
|
+
mock_results_db
|
102
|
+
|
103
|
+
lambda{ResultsDAL.new({}, "table_name", {:use_partitions => true, :field_name => "adgroup_id", :num_partitions => 101 })}.should raise_error(ResultsDAL::DBFatalError)
|
104
|
+
lambda{ResultsDAL.new({}, "table_name", {:use_partitions => true, :field_name => "adgroup_id", :num_partitions => 0 })}.should raise_error(ResultsDAL::DBFatalError)
|
105
|
+
end
|
106
|
+
|
107
|
+
it "should execute queries if 1 record is sent" do
|
108
|
+
mock_results_db
|
109
|
+
@db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
|
110
|
+
@db_mock.should_receive(:query).with("insert into table_name (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'42173','24892','447','9300','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);")
|
111
|
+
target = ResultsDAL.new({}, "table_name", nil)
|
112
|
+
columns = ["ad_id", "adgroup_id", "account_id", "campaign_id", "date", "spend", "clicks"]
|
113
|
+
rows = [{"ad_id"=>"42173", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2}]
|
114
|
+
|
115
|
+
target.insert_rows(rows,columns,["spend","clicks"])
|
116
|
+
|
117
|
+
end
|
118
|
+
|
119
|
+
it "should execute queries if 1 record is sent with manual partitioning by adgroup_id for 10 partitions" do
|
120
|
+
mock_results_db
|
121
|
+
@db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
|
122
|
+
@db_mock.should_receive(:query).with("insert into table_name_2 (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'42173','24892','447','9300','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);")
|
123
|
+
target = ResultsDAL.new({}, "table_name", {:use_partitions => true, :field_name => "adgroup_id", :num_partitions => 10})
|
124
|
+
columns = ["ad_id", "adgroup_id", "account_id", "campaign_id", "date", "spend", "clicks"]
|
125
|
+
rows = [{"ad_id"=>"42173", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2}]
|
126
|
+
|
127
|
+
target.insert_rows(rows,columns,["spend","clicks"])
|
128
|
+
|
129
|
+
end
|
130
|
+
|
131
|
+
it "should execute queries if 1 record is sent with manual partitioning by adgroup_id for 100 partitions" do
|
132
|
+
mock_results_db
|
133
|
+
@db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
|
134
|
+
@db_mock.should_receive(:query).with("insert into table_name_02 (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'42173','24802','447','9300','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);")
|
135
|
+
target = ResultsDAL.new({}, "table_name", {:use_partitions => true, :field_name => "adgroup_id", :num_partitions => 100})
|
136
|
+
columns = ["ad_id", "adgroup_id", "account_id", "campaign_id", "date", "spend", "clicks"]
|
137
|
+
rows = [{"ad_id"=>"42173", "adgroup_id"=>"24802", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2}]
|
138
|
+
|
139
|
+
target.insert_rows(rows,columns,["spend","clicks"])
|
140
|
+
|
141
|
+
end
|
142
|
+
|
143
|
+
it "should execute queries if 1 record is sent with manual partitioning by adgroup_id for 2 partitions when result is more than 2" do
|
144
|
+
mock_results_db
|
145
|
+
@db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
|
146
|
+
@db_mock.should_receive(:query).with("insert into table_name_0 (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'42173','24894','447','9300','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);")
|
147
|
+
target = ResultsDAL.new({}, "table_name", {:use_partitions => true, :field_name => "adgroup_id", :num_partitions => 2})
|
148
|
+
columns = ["ad_id", "adgroup_id", "account_id", "campaign_id", "date", "spend", "clicks"]
|
149
|
+
rows = [{"ad_id"=>"42173", "adgroup_id"=>"24894", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2}]
|
150
|
+
|
151
|
+
target.insert_rows(rows,columns,["spend","clicks"])
|
152
|
+
|
153
|
+
end
|
154
|
+
|
155
|
+
it "should raise InputError if partition field is missing" do
|
156
|
+
mock_results_db
|
157
|
+
@db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
|
158
|
+
|
159
|
+
target = ResultsDAL.new({}, "table_name", {:use_partitions => true, :field_name => "adgroup_id", :num_partitions => 2})
|
160
|
+
columns = ["ad_id", "adgroup_id", "account_id", "campaign_id", "date", "spend", "clicks"]
|
161
|
+
rows = [{"ad_id"=>"42173", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2}]
|
162
|
+
|
163
|
+
lambda {target.insert_rows(rows,columns,["spend","clicks"])}.should raise_error(ResultsDAL::InputError)
|
164
|
+
|
165
|
+
end
|
166
|
+
|
167
|
+
it "should execute 2 queries if 2 records are sent" do
|
168
|
+
mock_results_db
|
169
|
+
@db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
|
170
|
+
@db_mock.should_receive(:query).with("insert into table_name (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'42173','24892','447','9300','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").twice
|
171
|
+
target = ResultsDAL.new({}, "table_name", nil)
|
172
|
+
columns = ["ad_id", "adgroup_id", "account_id", "campaign_id", "date", "spend", "clicks"]
|
173
|
+
rows = [{"ad_id"=>"42173", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
|
174
|
+
{"ad_id"=>"42173", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2}]
|
175
|
+
|
176
|
+
target.insert_rows(rows,columns,["spend","clicks"])
|
177
|
+
|
178
|
+
end
|
179
|
+
|
180
|
+
it "should raise DBUpdateError exception if query failed" do
|
181
|
+
mock_results_db
|
182
|
+
@db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
|
183
|
+
target = ResultsDAL.new({}, "table_name", nil)
|
184
|
+
@db_mock.stub!(:query).and_raise(Mysql2::Error.new(""))
|
185
|
+
columns = ["ad_id", "adgroup_id", "account_id", "campaign_id", "date", "spend", "clicks"]
|
186
|
+
rows = [{"ad_id"=>"42173", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2}]
|
187
|
+
lambda {target.insert_rows(rows,columns,["spend","clicks"])}.should raise_error(ResultsDAL::DBUpdateError)
|
188
|
+
|
189
|
+
end
|
190
|
+
|
191
|
+
|
192
|
+
it "should rollback on insert exception when using transactions" do
|
193
|
+
mock_results_db
|
194
|
+
@db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
|
195
|
+
@db_mock.should_receive(:query).with("SET AUTOCOMMIT=0")
|
196
|
+
@db_mock.should_receive(:query).with("begin")
|
197
|
+
@db_mock.should_receive(:query).with("insert into table_name (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'42173','24892','447','9300','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").and_raise(Mysql2::Error.new(""))
|
198
|
+
@db_mock.should_receive(:query).with("rollback")
|
199
|
+
target = ResultsDAL.new({:use_transactions => true}, "table_name", nil)
|
200
|
+
columns = ["ad_id", "adgroup_id", "account_id", "campaign_id", "date", "spend", "clicks"]
|
201
|
+
rows = [{"ad_id"=>"42173", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2}]
|
202
|
+
lambda {target.insert_rows(rows,columns,["spend","clicks"])}.should raise_error(ResultsDAL::DBFatalError)
|
203
|
+
end
|
204
|
+
|
205
|
+
it "should execute 2 queries if 2 records are sent with transaction" do
|
206
|
+
mock_results_db
|
207
|
+
@db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
|
208
|
+
@db_mock.should_receive(:query).with("SET AUTOCOMMIT=0")
|
209
|
+
@db_mock.should_receive(:query).with("begin")
|
210
|
+
@db_mock.should_receive(:query).with("insert into table_name (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'42173','24892','447','9300','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").twice
|
211
|
+
@db_mock.should_receive(:query).with("commit")
|
212
|
+
target = ResultsDAL.new({:use_transactions => true}, "table_name", nil)
|
213
|
+
columns = ["ad_id", "adgroup_id", "account_id", "campaign_id", "date", "spend", "clicks"]
|
214
|
+
rows = [{"ad_id"=>"42173", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
|
215
|
+
{"ad_id"=>"42173", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2}]
|
216
|
+
|
217
|
+
target.insert_rows(rows,columns,["spend","clicks"])
|
218
|
+
|
219
|
+
end
|
220
|
+
|
221
|
+
it "should execute 2 queries by columns order" do
|
222
|
+
mock_results_db
|
223
|
+
@db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
|
224
|
+
@db_mock.should_receive(:query).with("SET AUTOCOMMIT=0")
|
225
|
+
@db_mock.should_receive(:query).with("begin")
|
226
|
+
@db_mock.should_receive(:query).with("insert into table_name (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'42173','24892','447','9300','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").once.ordered
|
227
|
+
|
228
|
+
@db_mock.should_receive(:query).with("insert into table_name (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'42175','24892','447','9300','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").once.ordered
|
229
|
+
@db_mock.should_receive(:query).with("commit")
|
230
|
+
target = ResultsDAL.new({:use_transactions => true}, "table_name", nil)
|
231
|
+
columns = ["ad_id", "adgroup_id", "account_id", "campaign_id", "date", "spend", "clicks"]
|
232
|
+
rows = [{"ad_id"=>"42175", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
|
233
|
+
{"ad_id"=>"42173", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2}]
|
234
|
+
|
235
|
+
target.insert_rows(rows,columns,["spend","clicks"])
|
236
|
+
|
237
|
+
end
|
238
|
+
|
239
|
+
it "should execute 2 queries by partition id and key columns order" do
|
240
|
+
mock_results_db
|
241
|
+
@db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
|
242
|
+
@db_mock.should_receive(:query).with("SET AUTOCOMMIT=0")
|
243
|
+
@db_mock.should_receive(:query).with("begin")
|
244
|
+
@db_mock.should_receive(:query).with("insert into table_name_2 (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'42173','24892','447','9300','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").once.ordered
|
245
|
+
@db_mock.should_receive(:query).with("insert into table_name_4 (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'1','30004','447','9200','2011-05-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").once.ordered
|
246
|
+
@db_mock.should_receive(:query).with("insert into table_name_4 (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'1','30004','447','9200','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").once.ordered
|
247
|
+
@db_mock.should_receive(:query).with("insert into table_name_4 (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'1','30004','447','9300','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").once.ordered
|
248
|
+
@db_mock.should_receive(:query).with("insert into table_name_4 (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'1','30004','447','9300','2011-09-27','1.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").once.ordered
|
249
|
+
@db_mock.should_receive(:query).with("insert into table_name_4 (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'10','40004','447','9300','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").once.ordered
|
250
|
+
|
251
|
+
@db_mock.should_receive(:query).with("commit")
|
252
|
+
target = ResultsDAL.new({:use_transactions => true}, "table_name", {:use_partitions => true, :field_name => "adgroup_id", :num_partitions => 10})
|
253
|
+
columns = ["ad_id", "adgroup_id", "account_id", "campaign_id", "date", "spend", "clicks"]
|
254
|
+
rows = [{"ad_id"=>"10", "adgroup_id"=>"40004", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
|
255
|
+
{"ad_id"=>"42173", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
|
256
|
+
{"ad_id"=>"1", "adgroup_id"=>"30004", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
|
257
|
+
{"ad_id"=>"1", "adgroup_id"=>"30004", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>1.0, "clicks"=>2},
|
258
|
+
{"ad_id"=>"1", "adgroup_id"=>"30004", "account_id"=>"447", "campaign_id"=>"9200", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
|
259
|
+
{"ad_id"=>"1", "adgroup_id"=>"30004", "account_id"=>"447", "campaign_id"=>"9200", "date"=>"2011-05-27", "spend"=>3.0, "clicks"=>2}]
|
260
|
+
|
261
|
+
target.insert_rows(rows,columns,["spend","clicks"])
|
262
|
+
end
|
263
|
+
|
264
|
+
it "should execute 2 queries by partition id and key columns order in small bulks" do
|
265
|
+
mock_results_db
|
266
|
+
@db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
|
267
|
+
@db_mock.should_receive(:query).with("SET AUTOCOMMIT=0")
|
268
|
+
@db_mock.should_receive(:query).with("begin")
|
269
|
+
@db_mock.should_receive(:query).with("insert into table_name_2 (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'42173','24892','447','9300','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").once.ordered
|
270
|
+
@db_mock.should_receive(:query).with("insert into table_name_4 (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'1','30004','447','9200','2011-05-27','3.0','2'),(NULL,'1','30004','447','9200','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").once.ordered
|
271
|
+
@db_mock.should_receive(:query).with("insert into table_name_4 (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'1','30004','447','9300','2011-09-27','3.0','2'),(NULL,'1','30004','447','9300','2011-09-27','1.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").once.ordered
|
272
|
+
@db_mock.should_receive(:query).with("insert into table_name_4 (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'10','40004','447','9300','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").once.ordered
|
273
|
+
|
274
|
+
@db_mock.should_receive(:query).with("commit")
|
275
|
+
target = ResultsDAL.new({:use_transactions => true}, "table_name", {:use_partitions => true, :field_name => "adgroup_id", :num_partitions => 10, :bulk_size => 2})
|
276
|
+
columns = ["ad_id", "adgroup_id", "account_id", "campaign_id", "date", "spend", "clicks"]
|
277
|
+
rows = [{"ad_id"=>"10", "adgroup_id"=>"40004", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
|
278
|
+
{"ad_id"=>"42173", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
|
279
|
+
{"ad_id"=>"1", "adgroup_id"=>"30004", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
|
280
|
+
{"ad_id"=>"1", "adgroup_id"=>"30004", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>1.0, "clicks"=>2},
|
281
|
+
{"ad_id"=>"1", "adgroup_id"=>"30004", "account_id"=>"447", "campaign_id"=>"9200", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
|
282
|
+
{"ad_id"=>"1", "adgroup_id"=>"30004", "account_id"=>"447", "campaign_id"=>"9200", "date"=>"2011-05-27", "spend"=>3.0, "clicks"=>2}]
|
283
|
+
|
284
|
+
target.insert_rows(rows,columns,["spend","clicks"])
|
285
|
+
end
|
286
|
+
|
287
|
+
it "should execute 2 queries by partition id and key columns order for large bulks" do
|
288
|
+
mock_results_db
|
289
|
+
@db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
|
290
|
+
@db_mock.should_receive(:query).with("SET AUTOCOMMIT=0")
|
291
|
+
@db_mock.should_receive(:query).with("begin")
|
292
|
+
@db_mock.should_receive(:query).with("insert into table_name_2 (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'42173','24892','447','9300','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").once.ordered
|
293
|
+
@db_mock.should_receive(:query).with("insert into table_name_4 (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'1','30004','447','9200','2011-05-27','3.0','2'),(NULL,'1','30004','447','9200','2011-09-27','3.0','2'),(NULL,'1','30004','447','9300','2011-09-27','3.0','2'),(NULL,'1','30004','447','9300','2011-09-27','1.0','2'),(NULL,'10','40004','447','9300','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").once.ordered
|
294
|
+
|
295
|
+
@db_mock.should_receive(:query).with("commit")
|
296
|
+
target = ResultsDAL.new({:use_transactions => true}, "table_name", {:use_partitions => true, :field_name => "adgroup_id", :num_partitions => 10, :bulk_size => 6})
|
297
|
+
columns = ["ad_id", "adgroup_id", "account_id", "campaign_id", "date", "spend", "clicks"]
|
298
|
+
rows = [{"ad_id"=>"10", "adgroup_id"=>"40004", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
|
299
|
+
{"ad_id"=>"42173", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
|
300
|
+
{"ad_id"=>"1", "adgroup_id"=>"30004", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
|
301
|
+
{"ad_id"=>"1", "adgroup_id"=>"30004", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>1.0, "clicks"=>2},
|
302
|
+
{"ad_id"=>"1", "adgroup_id"=>"30004", "account_id"=>"447", "campaign_id"=>"9200", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
|
303
|
+
{"ad_id"=>"1", "adgroup_id"=>"30004", "account_id"=>"447", "campaign_id"=>"9200", "date"=>"2011-05-27", "spend"=>3.0, "clicks"=>2}]
|
304
|
+
|
305
|
+
target.insert_rows(rows,columns,["spend","clicks"])
|
306
|
+
end
|
307
|
+
|
308
|
+
it "should execute queries without partition, with key columns and with duplicate actions columns order for large bulks" do
|
309
|
+
mock_results_db
|
310
|
+
@db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
|
311
|
+
@db_mock.should_receive(:query).with("SET AUTOCOMMIT=0")
|
312
|
+
@db_mock.should_receive(:query).with("begin")
|
313
|
+
@db_mock.should_receive(:query).with("insert into table_name (id,cid,category,action,label,date,event_count,event_sum,event_avg,event_min,event_max) values \n (NULL,'10','Facebook','Click','Feed2','2011-09-27','3','0','0','0','0'),(NULL,'10','Twitter','Click','Tweet1','2011-09-27','2','0','0','0','0'),(NULL,'10','Video','Play','Stop','2011-09-27','4','4.2','1.05','1','1.2'),(NULL,'100','Facebook','Click','Feed1','2011-09-27','2','0','0','0','0'),(NULL,'100','Facebook','Hoover',NULL,'2011-09-27','1','2.5','2.5','2.5','2.5'),(NULL,'1000','Facebook','Click','Feed1','2011-09-27','3','0','0','0','0') on duplicate key update event_count = event_count + values(event_count),event_sum = event_sum + values(event_sum),event_avg = (event_sum + values(event_sum))/(event_count + values(event_count)),event_min = LEAST(event_min,values(event_min)),event_max = GREATEST(event_max,values(event_max));").once.ordered
|
314
|
+
|
315
|
+
@db_mock.should_receive(:query).with("commit")
|
316
|
+
target = ResultsDAL.new({:use_transactions => true}, "table_name", {:use_partitions => false, :num_partitions => 0, :bulk_size => 6})
|
317
|
+
columns = ["cid","category","action","label","date","event_count","event_sum","event_avg","event_min","event_max"]
|
318
|
+
rows = [{"cid"=>"1000", "category"=>"Facebook", "action"=>"Click", "label"=>"Feed1", "date"=>"2011-09-27", "event_count"=>3, "event_sum"=>0, "event_avg"=>0, "event_min"=>0,"event_max"=>0},
|
319
|
+
{"cid"=>"100", "category"=>"Facebook", "action"=>"Click", "label"=>"Feed1", "date"=>"2011-09-27", "event_count"=>2, "event_sum"=>0, "event_avg"=>0, "event_min"=>0,"event_max"=>0},
|
320
|
+
{"cid"=>"100", "category"=>"Facebook", "action"=>"Hoover", "label"=>nil, "date"=>"2011-09-27", "event_count"=>1, "event_sum"=>2.5, "event_avg"=>2.5, "event_min"=>2.5,"event_max"=>2.5},
|
321
|
+
{"cid"=>"10", "category"=>"Facebook", "action"=>"Click", "label"=>"Feed2", "date"=>"2011-09-27", "event_count"=>3, "event_sum"=>0, "event_avg"=>0, "event_min"=>0,"event_max"=>0},
|
322
|
+
{"cid"=>"10", "category"=>"Twitter", "action"=>"Click", "label"=>"Tweet1", "date"=>"2011-09-27", "event_count"=>2, "event_sum"=>0, "event_avg"=>0, "event_min"=>0,"event_max"=>0},
|
323
|
+
{"cid"=>"10", "category"=>"Video", "action"=>"Play", "label"=>"Stop", "date"=>"2011-09-27", "event_count"=>4, "event_sum"=>4.2, "event_avg"=>1.05, "event_min"=>1,"event_max"=>1.2}]
|
324
|
+
target.insert_rows(rows,columns,["event_count","event_sum"],{"event_avg" => {"SUM" => "event_sum", "COUNT" => "event_count"}, "event_min" => "min", "event_max" => "max"})
|
325
|
+
end
|
326
|
+
|
327
|
+
it "should not use ids if requested" do
|
328
|
+
mock_results_db
|
329
|
+
@db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
|
330
|
+
@db_mock.should_receive(:query).with("SET AUTOCOMMIT=0")
|
331
|
+
@db_mock.should_receive(:query).with("begin")
|
332
|
+
@db_mock.should_receive(:query).with("insert into table_name_2 (ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n ('42173','24892','447','9300','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").once.ordered
|
333
|
+
@db_mock.should_receive(:query).with("insert into table_name_4 (ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n ('1','30004','447','9200','2011-05-27','3.0','2'),('1','30004','447','9200','2011-09-27','3.0','2'),('1','30004','447','9300','2011-09-27','3.0','2'),('1','30004','447','9300','2011-09-27','1.0','2'),('10','40004','447','9300','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").once.ordered
|
334
|
+
|
335
|
+
@db_mock.should_receive(:query).with("commit")
|
336
|
+
target = ResultsDAL.new({:use_transactions => true}, "table_name", {:use_partitions => true, :field_name => "adgroup_id", :num_partitions => 10, :bulk_size => 6, :use_ids => false})
|
337
|
+
columns = ["ad_id", "adgroup_id", "account_id", "campaign_id", "date", "spend", "clicks"]
|
338
|
+
rows = [{"ad_id"=>"10", "adgroup_id"=>"40004", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
|
339
|
+
{"ad_id"=>"42173", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
|
340
|
+
{"ad_id"=>"1", "adgroup_id"=>"30004", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
|
341
|
+
{"ad_id"=>"1", "adgroup_id"=>"30004", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>1.0, "clicks"=>2},
|
342
|
+
{"ad_id"=>"1", "adgroup_id"=>"30004", "account_id"=>"447", "campaign_id"=>"9200", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
|
343
|
+
{"ad_id"=>"1", "adgroup_id"=>"30004", "account_id"=>"447", "campaign_id"=>"9200", "date"=>"2011-05-27", "spend"=>3.0, "clicks"=>2}]
|
344
|
+
|
345
|
+
target.insert_rows(rows,columns,["spend","clicks"])
|
346
|
+
end
|
347
|
+
|
348
|
+
|
349
|
+
end
|
@@ -0,0 +1,125 @@
|
|
1
|
+
require 'rspec'
|
2
|
+
require 'redis'
|
3
|
+
require 'mysql2'
|
4
|
+
require_relative '../lib/ac-summarization-utils/revenue_calc_utils'
|
5
|
+
|
6
|
+
include ACSummarizationUtils
|
7
|
+
include ACSummarizationUtils::RevenueCalcUtils
|
8
|
+
|
9
|
+
describe "Revenue Calc Utils#" do
|
10
|
+
|
11
|
+
context "publisher_click_revenue#" do
|
12
|
+
|
13
|
+
it "should return rev share if publisher is cpc" do
|
14
|
+
revenue = 100
|
15
|
+
pub_rev_share = 0.1
|
16
|
+
pub_payment_type = 1
|
17
|
+
country = nil
|
18
|
+
calc_publisher_click_revenue(revenue, pub_rev_share, pub_payment_type, country).should == revenue*pub_rev_share
|
19
|
+
end
|
20
|
+
|
21
|
+
it "should return rev share if publisher is dynamic" do
|
22
|
+
revenue = 100
|
23
|
+
pub_rev_share = 0.1
|
24
|
+
pub_payment_type = 3
|
25
|
+
country = nil
|
26
|
+
calc_publisher_click_revenue(revenue, pub_rev_share, pub_payment_type, country).should == revenue*pub_rev_share
|
27
|
+
end
|
28
|
+
|
29
|
+
it "should return rev share if publisher is geo but country is not in geo countries" do
|
30
|
+
revenue = 100
|
31
|
+
pub_rev_share = 0.1
|
32
|
+
pub_payment_type = 4
|
33
|
+
country = "tanzania"
|
34
|
+
calc_publisher_click_revenue(revenue, pub_rev_share, pub_payment_type, country).should == revenue*pub_rev_share
|
35
|
+
end
|
36
|
+
|
37
|
+
it "should return 0 if publisher is cpm" do
|
38
|
+
revenue = 100
|
39
|
+
pub_rev_share = 0.1
|
40
|
+
pub_payment_type = 2
|
41
|
+
country = nil
|
42
|
+
calc_publisher_click_revenue(revenue, pub_rev_share, pub_payment_type, country).should == 0
|
43
|
+
end
|
44
|
+
|
45
|
+
it "should return 0 if publisher is geo and country is in geo countries" do
|
46
|
+
revenue = 100
|
47
|
+
pub_rev_share = 0.1
|
48
|
+
pub_payment_type = 2
|
49
|
+
country = "us"
|
50
|
+
calc_publisher_click_revenue(revenue, pub_rev_share, pub_payment_type, country).should == 0
|
51
|
+
end
|
52
|
+
|
53
|
+
end
|
54
|
+
|
55
|
+
context "publisher_cpm_revenue#" do
|
56
|
+
|
57
|
+
it "should return cpm share if publisher is cpm" do
|
58
|
+
imp_count = 5000
|
59
|
+
pub_cpm_share = 4
|
60
|
+
pub_payment_type = 2
|
61
|
+
country = nil
|
62
|
+
calc_publisher_cpm_revenue(imp_count, pub_cpm_share, pub_payment_type, country).should == (imp_count/1000)*pub_cpm_share
|
63
|
+
end
|
64
|
+
|
65
|
+
it "should return cpm share if publisher is dynamic" do
|
66
|
+
imp_count = 5000
|
67
|
+
pub_cpm_share = 4
|
68
|
+
pub_payment_type = 3
|
69
|
+
country = nil
|
70
|
+
calc_publisher_cpm_revenue(imp_count, pub_cpm_share, pub_payment_type, country).should == (imp_count/1000)*pub_cpm_share
|
71
|
+
end
|
72
|
+
|
73
|
+
it "should return cpm if publisher is geo and country is in geo countries" do
|
74
|
+
imp_count = 5000
|
75
|
+
pub_cpm_share = 4
|
76
|
+
pub_payment_type = 4
|
77
|
+
country = "us"
|
78
|
+
calc_publisher_cpm_revenue(imp_count, pub_cpm_share, pub_payment_type, country).should == (imp_count/1000)*pub_cpm_share
|
79
|
+
end
|
80
|
+
|
81
|
+
it "should return 0 if publisher is cpc" do
|
82
|
+
imp_count = 5000
|
83
|
+
pub_cpm_share = 4
|
84
|
+
pub_payment_type = 1
|
85
|
+
country = "us"
|
86
|
+
calc_publisher_cpm_revenue(imp_count, pub_cpm_share, pub_payment_type, country).should == 0
|
87
|
+
end
|
88
|
+
|
89
|
+
it "should return 0 if publisher is geo and country is not in geo countries" do
|
90
|
+
imp_count = 5000
|
91
|
+
pub_cpm_share = 4
|
92
|
+
pub_payment_type = 4
|
93
|
+
country = "tanzania"
|
94
|
+
calc_publisher_cpm_revenue(imp_count, pub_cpm_share, pub_payment_type, country).should == 0
|
95
|
+
end
|
96
|
+
|
97
|
+
end
|
98
|
+
|
99
|
+
context "Publisher US only revenue" do
|
100
|
+
|
101
|
+
it "should return 0 if country is not US" do
|
102
|
+
cpc = 0.54
|
103
|
+
country = "gb"
|
104
|
+
calc_publisher_us_click_revenue(cpc,country).should == 0
|
105
|
+
country = "oth"
|
106
|
+
calc_publisher_us_click_revenue(cpc,country).should == 0
|
107
|
+
end
|
108
|
+
|
109
|
+
it "should return 0 if NO country" do
|
110
|
+
cpc = 0.54
|
111
|
+
country = nil
|
112
|
+
calc_publisher_us_click_revenue(cpc,country).should == 0
|
113
|
+
end
|
114
|
+
|
115
|
+
it "should return CPC for country 'us'" do
|
116
|
+
cpc = 1.33
|
117
|
+
country = "us"
|
118
|
+
calc_publisher_us_click_revenue(cpc,country).should == 1.33
|
119
|
+
cpc = 0.88
|
120
|
+
country = "us"
|
121
|
+
calc_publisher_us_click_revenue(cpc,country).should == 0.88
|
122
|
+
end
|
123
|
+
end
|
124
|
+
|
125
|
+
end
|