ac-summarization-utils 0.0.47.rc1

data/lib/ac-summarization-utils/record.rb

@@ -0,0 +1,40 @@
+
+module ACSummarizationUtils
+
+  class Record
+
+    attr_accessor :required_fields
+    attr_accessor :group_fields
+
+    def initialize(input_hash)
+      @required_fields = input_hash[:required] if input_hash[:required]
+      @group_fields = input_hash[:group_fields] if input_hash[:group_fields]
+      @required_fields ||= {}
+      @group_fields ||= {}
+    end
+
+    def add_required_field(name, value)
+      @required_fields[name] = value
+    end
+
+    def add_group_field(name, value)
+      @group_fields[name] = value
+    end
+
+    def clone
+      cloned_record = Record.new({})
+      @required_fields.each do |name, val|
+        cloned_record.add_required_field(name, val)
+      end
+      @group_fields.each do |name, val|
+        cloned_record.add_group_field(name, val)
+      end
+      return cloned_record
+    end
+
+    def ==(other)
+      return (@required_fields == other.required_fields and  @group_fields == other.group_fields)
+    end
+
+  end
+end

data/lib/ac-summarization-utils/results_dal.rb

@@ -0,0 +1,189 @@
+#insert on duplicate update records to db (using table name, partitions data)
+require 'mysql2'
+
+module ACSummarizationUtils
+  class ResultsDAL
+
+    MYSQL_ACTION_TRANSLATOR = {"min" => "LEAST", "max" => "GREATEST" }
+
+    class DBFatalError < RuntimeError; end
+    class InputError < RuntimeError; end
+    class DBUpdateError < RuntimeError
+      attr_accessor :queries, :perform_rollback
+    end
+
+    def initialize(db_options, table_name, partition_options)
+      begin
+        @mysql_client = Mysql2::Client.new({:host => db_options[:host], :username => db_options[:user], :password => db_options[:password] , :database => db_options[:db_name], :reconnect => true})
+      rescue Exception => e
+        raise DBFatalError, e.message
+      end
+      @table_name = table_name
+      @use_partitions = partition_options.nil? ? false : partition_options[:use_partitions]
+      @bulk_size = (partition_options.nil? or partition_options[:bulk_size].nil?) ? 1 : partition_options[:bulk_size].to_i
+      @use_ids = true
+      @use_ids = partition_options[:use_ids] if !partition_options.nil? and !partition_options[:use_ids].nil?
+      if @use_partitions
+        @partition_field = partition_options[:field_name]
+        @partitions_number = partition_options[:num_partitions].to_i
+        @partition_prefixes = partition_options[:partition_prefixes] if partition_options[:partition_prefixes]
+	      @create_tables = (partition_options[:create_tables] ? true : false)
+        @create_table_query = "CREATE TABLE IF NOT EXISTS %TABLE_NAME% " + partition_options[:create_table_columns] if partition_options[:create_table_columns]
+        raise DBFatalError, "Bad partition arguments" if (@partition_field.nil? or @partitions_number.nil? or @partitions_number <= 0 or @partitions_number > 100)
+      else
+        @partitions_number = 1
+      end
+      @use_transactions = db_options[:use_transactions]
+      raise DBFatalError, "Results DB - table #{table_name} doesn't exist!!" unless (table_exists?(table_name) or (@create_tables and @create_table_query))
+    end
+
+    def table_exists?(table_name)
+      begin
+        query = "show tables like '#{table_name}%'"
+        result = @mysql_client.query(query)
+        return (result.nil? or result.size == 0 ? false : true)
+      rescue Exception => e
+        return false
+      end
+    end
+
+    def insert_rows(rows, columns ,duplicate_fields, duplicate_fields_actions=nil)
+      return if rows.nil? or rows.empty? or columns.nil? or columns.empty? 
+      if duplicate_fields.nil? or duplicate_fields.empty?
+        insert_queries, table_names = generate_unique_queries(rows,columns)
+      else
+        insert_queries, table_names = generate_queries(rows, columns ,duplicate_fields,duplicate_fields_actions)
+      end
+      current_index=0
+      begin
+        if @use_transactions
+           @mysql_client.query("SET AUTOCOMMIT=0")
+           @mysql_client.query("begin")
+        end
+        if (@create_tables)
+          create_tables_if_needed(table_names)
+        end
+        insert_queries.each_with_index do |insert_query,index|
+          current_index = index
+          @mysql_client.query(insert_query[:query])
+          insert_query[:status] = "success"
+        end
+        @mysql_client.query("commit") if @use_transactions
+      rescue Mysql2::Error => e
+        if @use_transactions
+          @mysql_client.query("rollback")
+          raise DBFatalError, "Perfoming rollback - #{e.message}"
+        else
+          db_update_error = DBUpdateError.new e.message
+          insert_queries[current_index][:status] = "failure"
+          db_update_error.queries = insert_queries
+          db_update_error.perform_rollback = false
+          raise db_update_error
+        end
+      end
+    end
+
+    private
+
+    def create_tables_if_needed(table_names)
+      return if @create_table_query.nil?
+      table_names.each do |table_name|
+        create_query =  @create_table_query.gsub('%TABLE_NAME%', table_name)
+        @mysql_client.query(create_query)
+      end
+    end
+
+    def generate_unique_queries(rows,columns)
+      insert_value_header = @use_ids ? "(id," : "("
+
+      insert_query_header = "insert into %TABLE_NAME% #{insert_value_header}#{columns.join(',')}) values \n"
+      table_queries, table_names = create_insert_values_per_partition(rows, columns)
+      queries = []
+
+      (0...@partitions_number).each do |part_id|
+          partition_id = @partitions_number > 10 ? '%02d' % part_id : part_id
+          next if table_queries[partition_id].nil?
+          data = table_queries[partition_id]
+          query_header = insert_query_header.gsub('%TABLE_NAME%', data[:table_name])
+ 	        current_size = data[:values].size
+          prev_index = 0
+          while current_size > 0 do
+            current_bulk_values = data[:values].slice(prev_index, @bulk_size).map { |v| v[:value] }
+            queries << {:query=>"#{query_header} #{current_bulk_values.join(',')};",:status=>"not run"}
+            current_size -= @bulk_size
+            prev_index += @bulk_size
+          end
+      end
+      return queries, table_names	
+    end
+
+    def generate_queries(rows, columns ,duplicate_fields, duplicate_fields_actions=nil)
+      insert_value_header = @use_ids ? "(id," : "("
+      insert_query_header = "insert into %TABLE_NAME% #{insert_value_header}#{columns.join(',')}) values \n"
+
+      duplicate_values = create_duplicate_values_statement(duplicate_fields,duplicate_fields_actions)
+      queries = []
+      table_queries, table_names = create_insert_values_per_partition(rows, columns)
+      key_columns = columns - duplicate_fields
+      
+      (0...@partitions_number).each do |part_id|
+          partition_id = @partitions_number > 10 ? '%02d' % part_id : part_id
+          next if table_queries[partition_id].nil?
+          data = table_queries[partition_id]
+          query_header = insert_query_header.gsub('%TABLE_NAME%', data[:table_name])
+          data[:values].sort_by! {|value| sort_key = []; key_columns.each { |col| sort_key << value[:original_row][col].to_s }; sort_key }
+          current_size = data[:values].size
+          prev_index = 0
+          while current_size > 0 do
+            current_bulk_values = data[:values].slice(prev_index, @bulk_size).map { |v| v[:value] }
+            queries << {:query=>"#{query_header} #{current_bulk_values.join(',')} on duplicate key update #{duplicate_values.join(',')};",:status=>"not run"}
+            current_size -= @bulk_size
+            prev_index += @bulk_size
+          end
+      end
+      return queries, table_names
+    end
+
+    def create_duplicate_values_statement(duplicate_fields,duplicate_fields_actions)
+      duplicate_values = duplicate_fields.map { |f| "#{f} = #{f} + values(#{f})" }
+      duplicate_fields_actions.each do |k,v|
+        if (MYSQL_ACTION_TRANSLATOR.has_key?(v))
+          duplicate_values << "#{k} = #{MYSQL_ACTION_TRANSLATOR[v]}(#{k},values(#{k}))"
+        else
+          sum = "(#{v['SUM']} + values(#{v['SUM']}))"
+          count = "(#{v['COUNT']} + values(#{v['COUNT']}))"
+          duplicate_values << "#{k} = #{sum}/#{count}"
+        end
+      end if !duplicate_fields_actions.nil?
+      return duplicate_values
+    end
+
+    def create_insert_values_per_partition(rows, columns)
+      table_queries = {}
+      partition_id = 0
+      table_names = Set.new
+      new_table_name = @table_name
+      value_header = @use_ids ? "(NULL," : "("
+      rows.each do |row|
+        values_array = columns.map { |c| row[c].nil? ? "NULL" : "'#{row[c].is_a?(String) ? @mysql_client.escape(row[c]) : row[c]}'" }
+        if @use_partitions
+          raise InputError, "Exception in manual partitioning by #{@partition_field} for row" if row[@partition_field].nil?
+          partition_id = row[@partition_field].to_i % @partitions_number
+          partition_id = '%02d' % partition_id if @partitions_number > 10
+          partition_suffix_arr = [partition_id]
+          if @partition_prefixes
+            partition_suffix_arr = @partition_prefixes.map { |prefix| row[prefix].to_s.gsub(/\s+/, '_') unless row[prefix].nil? }
+            partition_suffix_arr << partition_id
+          end
+	  
+          new_table_name = "#{@table_name}_#{partition_suffix_arr.join('_')}"
+          table_names << new_table_name
+        end
+        table_queries[partition_id] ||= { :table_name => new_table_name, :values => [] }
+        table_queries[partition_id][:values] << {:value =>"#{value_header}#{values_array.join(',')})", :original_row => row}
+      end
+      return table_queries, table_names
+    end
+
+  end
+end

data/lib/ac-summarization-utils/revenue_calc_utils.rb

@@ -0,0 +1,41 @@
+require 'set'
+
+module ACSummarizationUtils
+  module RevenueCalcUtils
+
+    PAYMENT_TYPES = { 1 => :rev_share, 2 => :cpm, 3 => :dynamic, 4 => :geo, 5 => :fixed, 6 => :min_rpc}
+    GEO_COUNTRIES = Set.new ["us"]
+
+    def calc_publisher_click_revenue(revenue, pub_rev_share, pub_payment_type, country)
+      result = 0
+      if should_calc_pub_revenue?(pub_payment_type,country)
+         result = pub_rev_share * revenue
+      end
+      return result
+    end
+
+    def calc_publisher_cpm_revenue(imp_count, pub_cpm_share, pub_payment_type, country)
+      result = 0
+      if !should_calc_pub_revenue?(pub_payment_type,country) or PAYMENT_TYPES[pub_payment_type] == :dynamic
+         result = (pub_cpm_share * imp_count)/1000
+      end
+      return result
+    end
+
+    def calc_publisher_us_click_revenue(cpc,country)
+      return 0 if country.nil?
+      cpc = ((GEO_COUNTRIES.include? country.strip) ? cpc : 0 )
+      return cpc
+    end
+
+    def is_cpm_geo?(country)
+      return false if country.nil?
+      return GEO_COUNTRIES.include? country.strip
+    end
+
+    def should_calc_pub_revenue?(pub_payment_type, country)
+      return (PAYMENT_TYPES[pub_payment_type] == :rev_share or  PAYMENT_TYPES[pub_payment_type] == :dynamic or (PAYMENT_TYPES[pub_payment_type] == :geo and !is_cpm_geo?(country)) or PAYMENT_TYPES[pub_payment_type] == :fixed or PAYMENT_TYPES[pub_payment_type] == :min_rpc )
+    end
+
+  end
+end

data/lib/ac-summarization-utils/summarization_utils.rb

@@ -0,0 +1,119 @@
+require_relative 'record'
+require_relative 'lookup_cache'
+require_relative 'errors_handler'
+
+module ACSummarizationUtils
+  module SummUtils
+
+    def initialize_resources(redis_options, lookup_db_options, results_db_options, db_partition_field, db_partition_prefixes=nil)
+      lookup_cache = LookupCache.connect(redis_options, lookup_db_options)
+      results_dal = ResultsDAL.new({:host => results_db_options[:host], :user => results_db_options[:user], :password => results_db_options[:password], :db_name => results_db_options[:name], :use_transactions => results_db_options[:use_transactions]},
+                                      results_db_options[:table_name],
+                                      { :use_partitions => results_db_options[:use_manual_partitions], :field_name => db_partition_field, :num_partitions => results_db_options[:num_partitions], :bulk_size => results_db_options[:bulk_size], :create_tables => results_db_options[:create_tables], :create_table_columns => results_db_options[:create_table_columns], :partition_prefixes => db_partition_prefixes})
+
+      return lookup_cache, results_dal
+    end
+
+    def create_records(logger, error_handler, messages ,&block)
+      if messages and messages.size > 0
+        records = []
+        input_error_messages = []
+        messages.each do |m|
+          begin
+            records.concat(block.call(downcase_hash(m)))
+          rescue LookupCache::InputError, ArgumentError =>e
+            #print_error(logger, "Caught exception of type #{e.class}, exception message - #{e.message}, #{e.backtrace} - sending erroneous message to a different queue")
+            input_error_messages << error_handler.generate_error_message(m,e)
+          end
+        end
+        return records, input_error_messages
+      end
+    end
+
+    #raises an exception if cannot parse date
+    def get_date(date)
+        return nil if date.nil?
+        begin
+            time_object =  date.is_a?(Fixnum) ? Time.at(date) : Time.parse(date)
+            return time_object.strftime("%Y-%m-%d")
+        rescue Exception => e
+            raise InputError, "Unable to parse time string: #{date} - message: #{e.message}"
+        end
+    end
+  
+    #downcase hash keys
+    def downcase_hash(input)
+      return nil if input.nil?
+      result = {}
+      input.each do |k, v|
+        result[k.downcase] = v
+      end
+      return result
+    end
+
+    #duplicate records according to specific field values
+    def duplicate_records(current_records, field, expanded_field_values)
+      return current_records if current_records.nil? or current_records.size == 0 or field.nil? or expanded_field_values.nil? or expanded_field_values.size == 0
+      additional_records = []
+      current_records.each do |new_record|
+        new_record.add_required_field(field, expanded_field_values.first)
+        expanded_field_values[1..-1].each do |expanded_value|
+          cloned_record = new_record.clone
+          cloned_record.add_required_field(field, expanded_value)
+          additional_records << cloned_record
+        end
+      end
+      current_records += additional_records
+      return current_records
+    end
+
+    #group_options is a array of this form: [ {:output_field_name => "", :method => "" , :input_field_name => "" }]
+    #example: [ {:output_field_name => "spend", :method => method(:sum), :input_field_name => "cpc"}]
+    def summarize_records(records, group_options)
+      return [] if records.nil? or records.empty?
+      unified_records = {}
+      records.each do |record|
+        unified_key = record.required_fields.to_a.join(";")
+        unified_records[unified_key] ||= record.required_fields
+        group_options.each do |group_operation|
+          group_operation[:method].call(unified_records[unified_key], record, group_operation[:input_field_name], group_operation[:output_field_name])
+        end
+      end
+      return unified_records.values
+    end
+
+    def sum(unified_record, record, input_field_name, output_field_name)
+      raise ArgumentError, "Unsupported field name for group function: #{input_field_name}" if record.group_fields[input_field_name].nil?
+      unified_record[output_field_name] ||= 0
+      unified_record[output_field_name] += record.group_fields[input_field_name].to_f
+    end
+
+    def count(unified_record, record, input_field_name, output_field_name)
+      raise ArgumentError, "Unsupported field name for group function: #{input_field_name}" if record.group_fields[input_field_name].nil?
+      unified_record[output_field_name] ||= 0
+      unified_record[output_field_name] += 1 if record.group_fields[input_field_name]
+    end
+
+    def minimum(unified_record, record, input_field_name, output_field_name)
+      raise ArgumentError, "Unsupported field name for group function: #{input_field_name}" if record.group_fields[input_field_name].nil?
+      unified_record[output_field_name] ||= 1000
+      unified_record[output_field_name] = [unified_record[output_field_name].to_f,record.group_fields[input_field_name].to_f].min
+    end
+
+    def maximum(unified_record, record, input_field_name, output_field_name)
+      raise ArgumentError, "Unsupported field name for group function: #{input_field_name}" if record.group_fields[input_field_name].nil?
+      unified_record[output_field_name] ||= 0
+      unified_record[output_field_name] = [unified_record[output_field_name].to_f,record.group_fields[input_field_name].to_f].max
+    end
+
+    def average(unified_record, record, input_field_name, output_field_name)
+      raise ArgumentError, "Unsupported field name for group function: #{input_field_name}" if record.group_fields[input_field_name].nil?
+      unified_record[output_field_name] ||= 0
+      sum(unified_record,record,input_field_name,output_field_name+"_sum")
+      count(unified_record,record,input_field_name,output_field_name+"_count")
+      unified_record[output_field_name] ||= 0
+      unified_record[output_field_name] = (( unified_record[output_field_name+"_count"] > 0 ) ? (unified_record[output_field_name+"_sum"] / unified_record[output_field_name+"_count"]) : 0.0 )
+    end
+
+  end
+end

data/lib/ac-summarization-utils/validation_utils.rb

@@ -0,0 +1,93 @@
+
+module ACSummarizationUtils
+  module ValidationUtils
+
+    ALLOWED_CLICK_TYPES = [1, 2, 5, 6, 1005]
+    REAL_CLICKS = [1, 3, 5, 11, 107, 108, 801, 805]
+    INVALID_CLICK_TYPE = -1
+    CPC = 1
+    CPV = 3
+
+    def common_click_validations(message)
+      click_type = message["clicktype"].nil? ? message["ct"] : message["clicktype"]
+      return [INVALID_CLICK_TYPE, -1] if click_type.to_s.empty? and (message["clickurl"].to_s.empty? or message["clickurl"] == 'NULL')
+      raise ArgumentError, "message required field click_type not ct doesn't exist" if click_type.nil?
+      valid = message["valid"].nil? ? message["validitytype"] : message["valid"]
+      raise ArgumentError, "message required field valid not validitytype doesn't exist" if valid.nil?
+      ["ad_id","advertiser_id","cpc"].each do |field_name|
+         raise ArgumentError, "message required field '#{field_name}' doesn't exist" if message[field_name].nil?
+      end
+      raise ArgumentError, "invalid ad id: #{message["ad_id"]}" unless message["ad_id"].to_s =~ /^[-]?[0-9]+$/ 
+      [click_type,valid]
+    end
+
+    def is_valid_real_click?(message, campaign_type)
+      click_type, valid = common_click_validations(message)
+      if (REAL_CLICKS.include?(click_type.to_i) or (1005 == click_type.to_i and campaign_type.to_i == CPC)) 
+        return (message["ad_id"].to_i != -1 and message["advertiser_id"].to_i == 8 and valid.to_i == 1)
+      end
+      return false
+    end
+
+    def is_valid_direct_click?(message)
+      click_type, valid = common_click_validations(message)
+      return (message["ad_id"].to_i != -1 and message["advertiser_id"].to_i == 8 and valid.to_i == 1 and
+               message["cpc"].to_f >= 0.0 and ALLOWED_CLICK_TYPES.include?(click_type.to_i))
+    end
+
+    def is_valid_direct_layer_impr?(message)
+      ["ad_id","advertiserid"].each do |field_name|
+         raise ArgumentError, "message required field '#{field_name}' doesn't exist" if message[field_name].nil?
+      end
+      return (message["ad_id"].to_i != -1 and message["advertiserid"].to_i == 8)
+    end
+    
+    def is_valid_conversion?(message)
+      ["adid","conversioneventid","publisherid"].each do |field_name|
+        raise ArgumentError, "message required field '#{field_name}' doesn't exist" if message[field_name].nil?
+      end
+      return (message["adid"].to_i != -1 and message["conversioneventid"].to_i > 0)
+    end
+    
+    def is_valid_dynamic_content_event?(message)
+      ["cid"].each do |field_name|
+        raise ArgumentError, "message required field '#{field_name}' doesn't exist" if message[field_name].nil?
+      end
+      return (message["cid"].to_i != -1)
+    end
+
+    def is_valid_layer_impr?(message)
+      ["ad_id","advertiserid","cpc"].each do |field_name|
+         raise ArgumentError, "message required field '#{field_name}' doesn't exist" if message[field_name].nil?
+      end
+      return message["cpc"].to_f >= 0.0
+    end
+
+    def is_valid_click?(message)
+      click_type, valid  = common_click_validations(message)
+      publisher_field = message["publisher_id"] || message["p"]
+      raise ArgumentError, "message required field 'publisher_id' or 'p' doesn't exist" if publisher_field.nil?
+
+      return ( valid.to_i == 1 and message["cpc"].to_f >= 0.0 and ALLOWED_CLICK_TYPES.include?(click_type.to_i))
+    end
+
+    def is_valid_impression?(message)
+      ["pid", "itid"].each do |field_name|
+         raise ArgumentError, "message required field '#{field_name}' doesn't exist" if message[field_name].nil?
+      end
+      return message["itid"].to_i == 3
+    end
+
+    def is_valid_inimage_report?(message)
+      ["value"].each do |field_name|
+        raise ArgumentError, "message required field '#{field_name}' doesn't exist" if message[field_name].nil?
+      end
+      value = message["value"]
+      ["account_id"].each do |field_name|
+        raise ArgumentError, "message required field '#{field_name}' doesn't exist" if value[field_name].nil?
+      end
+      return message["value"]["account_id"] != ""
+    end
+
+  end
+end