ac-summarization-utils 0.0.47.rc1

data/spec/results_dal_spec.rb

@@ -0,0 +1,349 @@
+require 'rspec'
+require 'mysql2'
+require_relative '../lib/ac-summarization-utils/results_dal'
+
+include ACSummarizationUtils
+
+describe "ResultsDAL#" do
+
+  def mock_results_db
+    @db_mock = mock("Mysql2::Client")
+    @db_mock.stub!(:escape) { |arg| arg }
+    Mysql2::Client.stub(:new).with(any_args()).and_return(@db_mock)
+  end
+
+  it "should not execute queries if no records are sent" do
+    mock_results_db
+    @db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
+
+    target = ResultsDAL.new({}, "table_name", nil)
+
+    target.insert_rows([], [], [])
+    target.insert_rows(nil, [], [])
+  end
+
+  it "should not execute  queries if no columns are sent" do
+    mock_results_db
+    @db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
+    rows = [{"ad_id"=>"42173", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2}]
+    target = ResultsDAL.new({}, "table_name", nil)
+
+    target.insert_rows(rows, nil, ["clicks"])
+    target.insert_rows(rows, [], ["clicks"])
+  end
+
+  it "should execute unique queries if no duplicate fields are sent" do
+    mock_results_db
+    @db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
+    rows = [{"ad_id"=>"42173", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2}]
+    @db_mock.should_receive(:query).with("insert into table_name (id,ad_id) values \n (NULL,'42173');").exactly(2).times
+    target = ResultsDAL.new({}, "table_name", nil)
+
+    target.insert_rows(rows, ["ad_id"], nil)
+    target.insert_rows(rows, ["ad_id"], [])
+  end
+
+  it "should add partition suffixes to table name if they are defined (unique insert)" do
+    mock_results_db
+    @db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
+    rows = [{"ad_id"=>"42173", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "hour" => 12, "day" => 27, "month" => 9}]
+    @db_mock.should_receive(:query).with("insert into table_name_27_9_12 (id,ad_id) values \n (NULL,'42173');").exactly(2).times
+    target = ResultsDAL.new({}, "table_name", {:use_partitions => true, :field_name => "hour", :num_partitions => 24, :partition_prefixes => ["day", "month"]})
+
+    target.insert_rows(rows, ["ad_id"], nil)
+    target.insert_rows(rows, ["ad_id"], [])
+  end
+
+  it "should add partition suffixes to table name if they are defined (duplicate insert)" do
+    mock_results_db
+    @db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
+    rows = [{"ad_id"=>"42173", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "hour" => 12, "day" => 27, "month" => 9, "clicks" => 2}]
+    @db_mock.should_receive(:query).with("insert into table_name_27_9_12 (id,ad_id) values \n (NULL,'42173') on duplicate key update clicks = clicks + values(clicks);")
+
+    target = ResultsDAL.new({}, "table_name", {:use_partitions => true, :field_name => "hour", :num_partitions => 24, :partition_prefixes => ["day", "month"]})
+
+    target.insert_rows(rows, ["ad_id"], ["clicks"])
+  end
+
+  it "should raise exception if table name doesn't exist" do
+    mock_results_db
+    @db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([])
+
+    lambda{ResultsDAL.new({}, "table_name", nil)}.should raise_error(ResultsDAL::DBFatalError)
+
+  end
+ 
+  it "should create tables if so specified" do
+    mock_results_db
+    @db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([])   
+    create_query = "( `id` int(11) NOT NULL auto_increment,  `adgroup_id` int(11) default NULL,  PRIMARY KEY  (`id`),) ENGINE=InnoDB AUTO_INCREMENT=39957 DEFAULT CHARSET=utf8"
+
+    rows = [{"ad_id"=>"42173", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "hour" => 12, "day" => 27, "month" => 9, "clicks" => 2},
+ 	    {"ad_id"=>"42172", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "hour" => 12, "day" => 27, "month" => 9, "clicks" => 1}]
+    @db_mock.should_receive(:query).with("CREATE TABLE IF NOT EXISTS table_name_27_9_12 "+create_query).and_return([])
+
+    @db_mock.should_receive(:query).with("insert into table_name_27_9_12 (id,ad_id) values \n (NULL,'42173') on duplicate key update clicks = clicks + values(clicks);")
+    @db_mock.should_receive(:query).with("insert into table_name_27_9_12 (id,ad_id) values \n (NULL,'42172') on duplicate key update clicks = clicks + values(clicks);")
+
+    target = ResultsDAL.new({}, "table_name", {:use_partitions => true, :field_name => "hour", :num_partitions => 24, :partition_prefixes => ["day", "month"], :create_tables => true, :create_table_columns => create_query})
+    target.insert_rows(rows, ["ad_id"], ["clicks"])
+  end
+
+  it "should raise exception if manual partitioning requested and relevant info missing" do
+    mock_results_db
+
+    lambda{ResultsDAL.new({}, "table_name", {:use_partitions => true})}.should raise_error(ResultsDAL::DBFatalError)
+    lambda{ResultsDAL.new({}, "table_name", {:use_partitions => true, :field_name => "adgroup_id"})}.should raise_error(ResultsDAL::DBFatalError)
+    lambda{ResultsDAL.new({}, "table_name", {:use_partitions => true, :num_partitions => 2})}.should raise_error(ResultsDAL::DBFatalError)
+  end
+
+  it "should raise exception if manual partitioning requested and partitions number is larger than 100 or lower/equal than 0" do
+    mock_results_db
+
+    lambda{ResultsDAL.new({}, "table_name", {:use_partitions => true, :field_name => "adgroup_id", :num_partitions => 101 })}.should raise_error(ResultsDAL::DBFatalError)
+    lambda{ResultsDAL.new({}, "table_name", {:use_partitions => true, :field_name => "adgroup_id", :num_partitions => 0 })}.should raise_error(ResultsDAL::DBFatalError)
+  end
+
+  it "should  execute queries if 1 record is sent" do
+    mock_results_db
+    @db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
+    @db_mock.should_receive(:query).with("insert into table_name (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'42173','24892','447','9300','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);")
+    target = ResultsDAL.new({}, "table_name", nil)
+    columns = ["ad_id", "adgroup_id", "account_id", "campaign_id", "date", "spend", "clicks"]
+    rows = [{"ad_id"=>"42173", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2}]
+
+    target.insert_rows(rows,columns,["spend","clicks"])
+
+  end
+
+  it "should  execute queries if 1 record is sent with manual partitioning by adgroup_id for 10 partitions" do
+    mock_results_db
+    @db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
+    @db_mock.should_receive(:query).with("insert into table_name_2 (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'42173','24892','447','9300','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);")
+    target = ResultsDAL.new({}, "table_name", {:use_partitions => true, :field_name => "adgroup_id", :num_partitions => 10})
+    columns = ["ad_id", "adgroup_id", "account_id", "campaign_id", "date", "spend", "clicks"]
+    rows = [{"ad_id"=>"42173", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2}]
+
+    target.insert_rows(rows,columns,["spend","clicks"])
+
+  end
+
+  it "should  execute queries if 1 record is sent with manual partitioning by adgroup_id for 100 partitions" do
+    mock_results_db
+    @db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
+    @db_mock.should_receive(:query).with("insert into table_name_02 (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'42173','24802','447','9300','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);")
+    target = ResultsDAL.new({}, "table_name", {:use_partitions => true, :field_name => "adgroup_id", :num_partitions => 100})
+    columns = ["ad_id", "adgroup_id", "account_id", "campaign_id", "date", "spend", "clicks"]
+    rows = [{"ad_id"=>"42173", "adgroup_id"=>"24802", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2}]
+
+    target.insert_rows(rows,columns,["spend","clicks"])
+
+  end
+
+  it "should  execute queries if 1 record is sent with manual partitioning by adgroup_id for 2 partitions when result is more than 2" do
+    mock_results_db
+    @db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
+    @db_mock.should_receive(:query).with("insert into table_name_0 (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'42173','24894','447','9300','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);")
+    target = ResultsDAL.new({}, "table_name", {:use_partitions => true, :field_name => "adgroup_id", :num_partitions => 2})
+    columns = ["ad_id", "adgroup_id", "account_id", "campaign_id", "date", "spend", "clicks"]
+    rows = [{"ad_id"=>"42173", "adgroup_id"=>"24894", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2}]
+
+    target.insert_rows(rows,columns,["spend","clicks"])
+
+  end
+
+  it "should raise InputError if partition field is missing" do
+    mock_results_db
+    @db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
+
+    target = ResultsDAL.new({}, "table_name", {:use_partitions => true, :field_name => "adgroup_id", :num_partitions => 2})
+    columns = ["ad_id", "adgroup_id", "account_id", "campaign_id", "date", "spend", "clicks"]
+    rows = [{"ad_id"=>"42173", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2}]
+
+    lambda {target.insert_rows(rows,columns,["spend","clicks"])}.should raise_error(ResultsDAL::InputError)
+
+  end
+
+  it "should  execute 2 queries if 2 records are sent" do
+    mock_results_db
+    @db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
+    @db_mock.should_receive(:query).with("insert into table_name (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'42173','24892','447','9300','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").twice
+    target = ResultsDAL.new({}, "table_name", nil)
+    columns = ["ad_id", "adgroup_id", "account_id", "campaign_id", "date", "spend", "clicks"]
+    rows = [{"ad_id"=>"42173", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
+            {"ad_id"=>"42173", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2}]
+
+    target.insert_rows(rows,columns,["spend","clicks"])
+
+  end
+
+  it "should raise DBUpdateError exception if query failed" do
+    mock_results_db
+    @db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
+    target = ResultsDAL.new({}, "table_name", nil)
+    @db_mock.stub!(:query).and_raise(Mysql2::Error.new(""))
+    columns = ["ad_id", "adgroup_id", "account_id", "campaign_id", "date", "spend", "clicks"]
+    rows = [{"ad_id"=>"42173", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2}]
+    lambda {target.insert_rows(rows,columns,["spend","clicks"])}.should raise_error(ResultsDAL::DBUpdateError)
+
+  end
+
+
+  it "should  rollback on insert exception when using transactions" do
+    mock_results_db
+    @db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
+    @db_mock.should_receive(:query).with("SET AUTOCOMMIT=0")
+    @db_mock.should_receive(:query).with("begin")
+    @db_mock.should_receive(:query).with("insert into table_name (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'42173','24892','447','9300','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").and_raise(Mysql2::Error.new(""))
+    @db_mock.should_receive(:query).with("rollback")
+    target = ResultsDAL.new({:use_transactions => true}, "table_name", nil)
+    columns = ["ad_id", "adgroup_id", "account_id", "campaign_id", "date", "spend", "clicks"]
+    rows = [{"ad_id"=>"42173", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2}]
+    lambda {target.insert_rows(rows,columns,["spend","clicks"])}.should raise_error(ResultsDAL::DBFatalError)
+  end
+
+  it "should  execute 2 queries if 2 records are sent with transaction" do
+    mock_results_db
+    @db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
+    @db_mock.should_receive(:query).with("SET AUTOCOMMIT=0")
+    @db_mock.should_receive(:query).with("begin")
+    @db_mock.should_receive(:query).with("insert into table_name (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'42173','24892','447','9300','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").twice
+    @db_mock.should_receive(:query).with("commit")
+    target = ResultsDAL.new({:use_transactions => true}, "table_name", nil)
+    columns = ["ad_id", "adgroup_id", "account_id", "campaign_id", "date", "spend", "clicks"]
+    rows = [{"ad_id"=>"42173", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
+            {"ad_id"=>"42173", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2}]
+
+    target.insert_rows(rows,columns,["spend","clicks"])
+
+  end
+
+  it "should  execute 2 queries by columns order" do
+    mock_results_db
+    @db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
+    @db_mock.should_receive(:query).with("SET AUTOCOMMIT=0")
+    @db_mock.should_receive(:query).with("begin")
+    @db_mock.should_receive(:query).with("insert into table_name (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'42173','24892','447','9300','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").once.ordered
+ 
+    @db_mock.should_receive(:query).with("insert into table_name (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'42175','24892','447','9300','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").once.ordered
+    @db_mock.should_receive(:query).with("commit")
+    target = ResultsDAL.new({:use_transactions => true}, "table_name", nil)
+    columns = ["ad_id", "adgroup_id", "account_id", "campaign_id", "date", "spend", "clicks"]
+    rows = [{"ad_id"=>"42175", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
+            {"ad_id"=>"42173", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2}]
+
+    target.insert_rows(rows,columns,["spend","clicks"])
+
+  end
+
+  it "should  execute 2 queries by partition id and key columns order" do
+    mock_results_db
+    @db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
+    @db_mock.should_receive(:query).with("SET AUTOCOMMIT=0")
+    @db_mock.should_receive(:query).with("begin")
+    @db_mock.should_receive(:query).with("insert into table_name_2 (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'42173','24892','447','9300','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").once.ordered
+    @db_mock.should_receive(:query).with("insert into table_name_4 (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'1','30004','447','9200','2011-05-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").once.ordered
+    @db_mock.should_receive(:query).with("insert into table_name_4 (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'1','30004','447','9200','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").once.ordered
+    @db_mock.should_receive(:query).with("insert into table_name_4 (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'1','30004','447','9300','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").once.ordered
+    @db_mock.should_receive(:query).with("insert into table_name_4 (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'1','30004','447','9300','2011-09-27','1.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").once.ordered
+    @db_mock.should_receive(:query).with("insert into table_name_4 (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'10','40004','447','9300','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").once.ordered
+
+    @db_mock.should_receive(:query).with("commit")
+    target = ResultsDAL.new({:use_transactions => true}, "table_name", {:use_partitions => true, :field_name => "adgroup_id", :num_partitions => 10})
+    columns = ["ad_id", "adgroup_id", "account_id", "campaign_id", "date", "spend", "clicks"]
+    rows = [{"ad_id"=>"10", "adgroup_id"=>"40004", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
+            {"ad_id"=>"42173", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
+            {"ad_id"=>"1", "adgroup_id"=>"30004", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
+            {"ad_id"=>"1", "adgroup_id"=>"30004", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>1.0, "clicks"=>2},
+            {"ad_id"=>"1", "adgroup_id"=>"30004", "account_id"=>"447", "campaign_id"=>"9200", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
+            {"ad_id"=>"1", "adgroup_id"=>"30004", "account_id"=>"447", "campaign_id"=>"9200", "date"=>"2011-05-27", "spend"=>3.0, "clicks"=>2}]
+
+    target.insert_rows(rows,columns,["spend","clicks"])
+  end
+
+  it "should  execute 2 queries by partition id and key columns order in small bulks" do
+    mock_results_db
+    @db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
+    @db_mock.should_receive(:query).with("SET AUTOCOMMIT=0")
+    @db_mock.should_receive(:query).with("begin")
+    @db_mock.should_receive(:query).with("insert into table_name_2 (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'42173','24892','447','9300','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").once.ordered
+    @db_mock.should_receive(:query).with("insert into table_name_4 (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'1','30004','447','9200','2011-05-27','3.0','2'),(NULL,'1','30004','447','9200','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").once.ordered
+    @db_mock.should_receive(:query).with("insert into table_name_4 (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'1','30004','447','9300','2011-09-27','3.0','2'),(NULL,'1','30004','447','9300','2011-09-27','1.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").once.ordered
+    @db_mock.should_receive(:query).with("insert into table_name_4 (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'10','40004','447','9300','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").once.ordered
+
+    @db_mock.should_receive(:query).with("commit")
+    target = ResultsDAL.new({:use_transactions => true}, "table_name", {:use_partitions => true, :field_name => "adgroup_id", :num_partitions => 10, :bulk_size => 2})
+    columns = ["ad_id", "adgroup_id", "account_id", "campaign_id", "date", "spend", "clicks"]
+    rows = [{"ad_id"=>"10", "adgroup_id"=>"40004", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
+            {"ad_id"=>"42173", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
+            {"ad_id"=>"1", "adgroup_id"=>"30004", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
+            {"ad_id"=>"1", "adgroup_id"=>"30004", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>1.0, "clicks"=>2},
+            {"ad_id"=>"1", "adgroup_id"=>"30004", "account_id"=>"447", "campaign_id"=>"9200", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
+            {"ad_id"=>"1", "adgroup_id"=>"30004", "account_id"=>"447", "campaign_id"=>"9200", "date"=>"2011-05-27", "spend"=>3.0, "clicks"=>2}]
+
+    target.insert_rows(rows,columns,["spend","clicks"])
+  end
+
+  it "should  execute 2 queries by partition id and key columns order for large bulks" do
+    mock_results_db
+    @db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
+    @db_mock.should_receive(:query).with("SET AUTOCOMMIT=0")
+    @db_mock.should_receive(:query).with("begin")
+    @db_mock.should_receive(:query).with("insert into table_name_2 (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'42173','24892','447','9300','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").once.ordered
+    @db_mock.should_receive(:query).with("insert into table_name_4 (id,ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n (NULL,'1','30004','447','9200','2011-05-27','3.0','2'),(NULL,'1','30004','447','9200','2011-09-27','3.0','2'),(NULL,'1','30004','447','9300','2011-09-27','3.0','2'),(NULL,'1','30004','447','9300','2011-09-27','1.0','2'),(NULL,'10','40004','447','9300','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").once.ordered
+
+    @db_mock.should_receive(:query).with("commit")
+    target = ResultsDAL.new({:use_transactions => true}, "table_name", {:use_partitions => true, :field_name => "adgroup_id", :num_partitions => 10, :bulk_size => 6})
+    columns = ["ad_id", "adgroup_id", "account_id", "campaign_id", "date", "spend", "clicks"]
+    rows = [{"ad_id"=>"10", "adgroup_id"=>"40004", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
+            {"ad_id"=>"42173", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
+            {"ad_id"=>"1", "adgroup_id"=>"30004", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
+            {"ad_id"=>"1", "adgroup_id"=>"30004", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>1.0, "clicks"=>2},
+            {"ad_id"=>"1", "adgroup_id"=>"30004", "account_id"=>"447", "campaign_id"=>"9200", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
+            {"ad_id"=>"1", "adgroup_id"=>"30004", "account_id"=>"447", "campaign_id"=>"9200", "date"=>"2011-05-27", "spend"=>3.0, "clicks"=>2}]
+
+    target.insert_rows(rows,columns,["spend","clicks"])
+  end
+
+  it "should execute queries without partition, with key columns and with duplicate actions columns order for large bulks" do
+    mock_results_db
+    @db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
+    @db_mock.should_receive(:query).with("SET AUTOCOMMIT=0")
+    @db_mock.should_receive(:query).with("begin")
+    @db_mock.should_receive(:query).with("insert into table_name (id,cid,category,action,label,date,event_count,event_sum,event_avg,event_min,event_max) values \n (NULL,'10','Facebook','Click','Feed2','2011-09-27','3','0','0','0','0'),(NULL,'10','Twitter','Click','Tweet1','2011-09-27','2','0','0','0','0'),(NULL,'10','Video','Play','Stop','2011-09-27','4','4.2','1.05','1','1.2'),(NULL,'100','Facebook','Click','Feed1','2011-09-27','2','0','0','0','0'),(NULL,'100','Facebook','Hoover',NULL,'2011-09-27','1','2.5','2.5','2.5','2.5'),(NULL,'1000','Facebook','Click','Feed1','2011-09-27','3','0','0','0','0') on duplicate key update event_count = event_count + values(event_count),event_sum = event_sum + values(event_sum),event_avg = (event_sum + values(event_sum))/(event_count + values(event_count)),event_min = LEAST(event_min,values(event_min)),event_max = GREATEST(event_max,values(event_max));").once.ordered
+
+    @db_mock.should_receive(:query).with("commit")
+    target = ResultsDAL.new({:use_transactions => true}, "table_name", {:use_partitions => false, :num_partitions => 0, :bulk_size => 6})
+    columns = ["cid","category","action","label","date","event_count","event_sum","event_avg","event_min","event_max"]
+    rows = [{"cid"=>"1000", "category"=>"Facebook", "action"=>"Click", "label"=>"Feed1", "date"=>"2011-09-27", "event_count"=>3, "event_sum"=>0, "event_avg"=>0, "event_min"=>0,"event_max"=>0},
+            {"cid"=>"100", "category"=>"Facebook", "action"=>"Click", "label"=>"Feed1", "date"=>"2011-09-27", "event_count"=>2, "event_sum"=>0, "event_avg"=>0, "event_min"=>0,"event_max"=>0},
+            {"cid"=>"100", "category"=>"Facebook", "action"=>"Hoover", "label"=>nil, "date"=>"2011-09-27", "event_count"=>1, "event_sum"=>2.5, "event_avg"=>2.5, "event_min"=>2.5,"event_max"=>2.5},
+            {"cid"=>"10", "category"=>"Facebook", "action"=>"Click", "label"=>"Feed2", "date"=>"2011-09-27", "event_count"=>3, "event_sum"=>0, "event_avg"=>0, "event_min"=>0,"event_max"=>0},
+            {"cid"=>"10", "category"=>"Twitter", "action"=>"Click", "label"=>"Tweet1", "date"=>"2011-09-27", "event_count"=>2, "event_sum"=>0, "event_avg"=>0, "event_min"=>0,"event_max"=>0},
+            {"cid"=>"10", "category"=>"Video", "action"=>"Play", "label"=>"Stop", "date"=>"2011-09-27", "event_count"=>4, "event_sum"=>4.2, "event_avg"=>1.05, "event_min"=>1,"event_max"=>1.2}]
+    target.insert_rows(rows,columns,["event_count","event_sum"],{"event_avg" => {"SUM" => "event_sum", "COUNT" => "event_count"}, "event_min" => "min", "event_max" => "max"})
+  end
+
+  it "should not use ids if requested" do
+    mock_results_db
+    @db_mock.should_receive(:query).with("show tables like 'table_name%'").and_return([{"table_name" => "blah"}])
+    @db_mock.should_receive(:query).with("SET AUTOCOMMIT=0")
+    @db_mock.should_receive(:query).with("begin")
+    @db_mock.should_receive(:query).with("insert into table_name_2 (ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n ('42173','24892','447','9300','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").once.ordered
+    @db_mock.should_receive(:query).with("insert into table_name_4 (ad_id,adgroup_id,account_id,campaign_id,date,spend,clicks) values \n ('1','30004','447','9200','2011-05-27','3.0','2'),('1','30004','447','9200','2011-09-27','3.0','2'),('1','30004','447','9300','2011-09-27','3.0','2'),('1','30004','447','9300','2011-09-27','1.0','2'),('10','40004','447','9300','2011-09-27','3.0','2') on duplicate key update spend = spend + values(spend),clicks = clicks + values(clicks);").once.ordered
+
+    @db_mock.should_receive(:query).with("commit")
+    target = ResultsDAL.new({:use_transactions => true}, "table_name", {:use_partitions => true, :field_name => "adgroup_id", :num_partitions => 10, :bulk_size => 6, :use_ids => false})
+    columns = ["ad_id", "adgroup_id", "account_id", "campaign_id", "date", "spend", "clicks"]
+    rows = [{"ad_id"=>"10", "adgroup_id"=>"40004", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
+            {"ad_id"=>"42173", "adgroup_id"=>"24892", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
+            {"ad_id"=>"1", "adgroup_id"=>"30004", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
+            {"ad_id"=>"1", "adgroup_id"=>"30004", "account_id"=>"447", "campaign_id"=>"9300", "date"=>"2011-09-27", "spend"=>1.0, "clicks"=>2},
+            {"ad_id"=>"1", "adgroup_id"=>"30004", "account_id"=>"447", "campaign_id"=>"9200", "date"=>"2011-09-27", "spend"=>3.0, "clicks"=>2},
+            {"ad_id"=>"1", "adgroup_id"=>"30004", "account_id"=>"447", "campaign_id"=>"9200", "date"=>"2011-05-27", "spend"=>3.0, "clicks"=>2}]
+
+    target.insert_rows(rows,columns,["spend","clicks"])
+  end
+
+
+end

data/spec/revenue_calc_utils_spec.rb

@@ -0,0 +1,125 @@
+require 'rspec'
+require 'redis'
+require 'mysql2'
+require_relative '../lib/ac-summarization-utils/revenue_calc_utils'
+
+include ACSummarizationUtils
+include ACSummarizationUtils::RevenueCalcUtils
+
+describe "Revenue Calc Utils#" do
+
+  context "publisher_click_revenue#" do
+
+    it "should return rev share if publisher is cpc" do
+      revenue = 100
+      pub_rev_share = 0.1
+      pub_payment_type = 1
+      country = nil
+      calc_publisher_click_revenue(revenue, pub_rev_share, pub_payment_type, country).should == revenue*pub_rev_share
+    end
+
+    it "should return rev share if publisher is dynamic" do
+      revenue = 100
+      pub_rev_share = 0.1
+      pub_payment_type = 3
+      country = nil
+      calc_publisher_click_revenue(revenue, pub_rev_share, pub_payment_type, country).should == revenue*pub_rev_share
+    end
+
+    it "should return rev share if publisher is geo but country is not in geo countries" do
+      revenue = 100
+      pub_rev_share = 0.1
+      pub_payment_type = 4
+      country = "tanzania"
+      calc_publisher_click_revenue(revenue, pub_rev_share, pub_payment_type, country).should == revenue*pub_rev_share
+    end
+
+    it "should return 0 if publisher is cpm" do
+      revenue = 100
+      pub_rev_share = 0.1
+      pub_payment_type = 2
+      country = nil
+      calc_publisher_click_revenue(revenue, pub_rev_share, pub_payment_type, country).should == 0
+    end
+
+    it "should return 0 if publisher is geo and country is in geo countries" do
+      revenue = 100
+      pub_rev_share = 0.1
+      pub_payment_type = 2
+      country = "us"
+      calc_publisher_click_revenue(revenue, pub_rev_share, pub_payment_type, country).should == 0
+    end
+
+  end
+
+  context "publisher_cpm_revenue#" do
+
+    it "should return cpm share if publisher is cpm" do
+      imp_count = 5000
+      pub_cpm_share = 4
+      pub_payment_type = 2
+      country = nil
+      calc_publisher_cpm_revenue(imp_count, pub_cpm_share, pub_payment_type, country).should == (imp_count/1000)*pub_cpm_share
+    end
+
+    it "should return cpm share if publisher is dynamic" do
+      imp_count = 5000
+      pub_cpm_share = 4
+      pub_payment_type = 3
+      country = nil
+      calc_publisher_cpm_revenue(imp_count, pub_cpm_share, pub_payment_type, country).should == (imp_count/1000)*pub_cpm_share
+    end
+
+    it "should return cpm if publisher is geo and country is in geo countries" do
+      imp_count = 5000
+      pub_cpm_share = 4
+      pub_payment_type = 4
+      country = "us"
+      calc_publisher_cpm_revenue(imp_count, pub_cpm_share, pub_payment_type, country).should == (imp_count/1000)*pub_cpm_share
+    end
+
+    it "should return 0 if publisher is cpc" do
+      imp_count = 5000
+      pub_cpm_share = 4
+      pub_payment_type = 1
+      country = "us"
+      calc_publisher_cpm_revenue(imp_count, pub_cpm_share, pub_payment_type, country).should == 0
+    end
+
+    it "should return 0 if publisher is geo and country is not in geo countries" do
+      imp_count = 5000
+      pub_cpm_share = 4
+      pub_payment_type = 4
+      country = "tanzania"
+      calc_publisher_cpm_revenue(imp_count, pub_cpm_share, pub_payment_type, country).should == 0
+    end
+
+  end
+
+  context "Publisher US only revenue" do
+
+    it "should return 0 if country is not US" do
+      cpc = 0.54
+      country = "gb"
+      calc_publisher_us_click_revenue(cpc,country).should == 0
+      country = "oth"
+      calc_publisher_us_click_revenue(cpc,country).should == 0
+    end
+
+    it "should return 0 if NO country" do
+      cpc = 0.54
+      country = nil
+      calc_publisher_us_click_revenue(cpc,country).should == 0
+    end
+
+    it "should return CPC for country 'us'" do
+      cpc = 1.33
+      country = "us"
+      calc_publisher_us_click_revenue(cpc,country).should == 1.33
+      cpc = 0.88
+      country = "us"
+      calc_publisher_us_click_revenue(cpc,country).should == 0.88
+    end
+  end
+
+end