geoptima 0.1.3 → 0.1.4

data/bin/show_geoptima

@@ -6,8 +6,9 @@ $: << '../lib'
 
 require 'date'
 require 'geoptima'
+require 'geoptima/options'
 
-Geoptima::assert_version("0.1.3")
+Geoptima::assert_version("0.1.4")
 
 $debug=false
 
@@ -24,29 +25,89 @@ $files = Geoptima::Options.process_args do |option|
   option.m {$map_headers = true}
   option.a {$combine_all = true}
   option.l {$more_headers = true}
+  option.P {$export_prefix = ARGV.shift}
   option.E {$event_names += ARGV.shift.split(/[\,\;\:\.]+/)}
-  option.T do
-    $time_range = Geoptima::DateRange.from ARGV.shift
-  end
+  option.T {$time_range = Geoptima::DateRange.from ARGV.shift}
   option.L {$print_limit = [1,ARGV.shift.to_i].max}
-
-  option.t {$time_split = true}
-  option.D {$export_dir = ARGV.shift}
-  option.N {$merged_name = ARGV.shift}
-  option.S {$specfile = ARGV.shift}
-  option.P {$diversity = ARGV.shift.to_f}
-  option.W {$chart_width = ARGV.shift.to_i}
-  option.T do
-    $time_range = Geoptima::DateRange.from ARGV.shift
-  end
+  option.M {$mapfile = ARGV.shift}
 end.map do |file|
   File.exist?(file) ? file : puts("No such file: #{file}")
 end.compact
 
+class HeaderMap
+  attr_reader :prefix, :name, :event
+  attr_accessor :columns
+  def initialize(prefix,name,event)
+    @prefix = prefix
+    @name = name
+    @event = event
+    @columns = []
+  end
+  def mk_known(header)
+    puts "Creating column mappings for headers: #{header}" if($debug)
+    @col_indices = {}
+    columns.each do |col|
+      c = (col[1] && col[1].gsub(/\?/,'')).to_s
+      if c.length>0
+        @col_indices[c] = header.index(c)
+        puts "\tMade column mapping: #{c} --> #{header.index(c)}" if($debug)
+      end
+    end
+  end
+  def map_fields(header,fields)
+    @scenario_counter ||= 0
+    mk_known(header) unless @col_indices
+    @columns.map do |column|
+      if column[1] =~ /SCENARIO_COUNTER/
+        @scenario_counter += 1
+      else
+        index = @col_indices[column[1]]
+        puts "Found mapping #{column} -> #{index} -> #{index && fields[index]}" if($debug)
+        index && fields[index]
+      end
+    end
+  end
+end
+
+if $mapfile
+  $header_maps = []
+  current_map = nil
+  prefix = $mapfile.split(/\./)[0]
+  File.open($mapfile).each do |line|
+    line.chomp!
+    next if line =~ /^\s*#/
+    next if line.length < 2
+    if line =~ /^\[(\w+)\]\t(\w+)/
+      current_map = HeaderMap.new(prefix,$1,$2)
+      $header_maps << current_map
+    elsif current_map
+      current_map.columns << line.chomp.split(/\t/)[0..1]
+    else
+      puts "Invalid header map line: #{line}"
+    end
+  end
+end
+
+def show_header_maps
+  if $header_maps
+    puts "Using #{$header_maps.length} header maps:"
+    $header_maps.each do |hm|
+      puts "\t[#{hm.name}] (#{hm.event})"
+      if $debug
+        hm.columns.each do |hc|
+          puts "\t\t#{hc.map{|c| (c+' '*30)[0..30]}.join("\t-->\t")}"
+        end
+      else
+        puts "\t\t#{hm.columns.map{|hc| hc[0]}.join(', ')}"
+      end
+    end
+  end
+end
+
 $help = true if($files.length < 1)
 if $help
   puts <<EOHELP
-Usage: show_geoptima <-dpvxomlsah> <-L limit> <-E types> <-T min,max> file <files>
+Usage: show_geoptima <-dpvxomlsah> <-L limit> <-E types> <-T min,max> <-M mapfile> file <files>
   -d  debug mode (output more context during processing) #{cw $debug}
   -p  print mode (print out final results to console) #{cw $print}
   -v  verbose mode (output extra information to console) #{cw $verbose}
@@ -57,14 +118,19 @@ Usage: show_geoptima <-dpvxomlsah> <-L limit> <-E types> <-T min,max> file <file
   -s  seperate the export files by event type #{cw $seperate}
   -a  combine all IMEI's into a single dataset #{cw $combine_all}
   -h  show this help
+  -P  prefix for exported files (default: ''; current: #{$export_prefix})
   -E  comma-seperated list of event types to show and export (default: all; current: #{$event_names.join(',')})
   -T  time range to limit results to (default: all; current: #{$time_range})
   -L  limit verbose output to specific number of lines #{cw $print_limit}
+  -M  mapfile of normal->altered header names: #{$mapfile}
 EOHELP
+  show_header_maps
   exit 0
 end
 
 $verbose = $verbose || $debug
+show_header_maps if($verbose)
+
 $datasets = Geoptima::Dataset.make_datasets($files, :locate => true, :time_range => $time_range, :combine_all => $combine_all)
 
 class Export
@@ -74,13 +140,18 @@ class Export
     @imei = imei
     @names = names
     if $export
-      if $seperate
+      if $header_maps
+        @files = $header_maps.inject({}) do |a,hm|
+          a[hm.event] = File.open("#{$export_prefix}#{imei}_#{hm.prefix}_#{hm.name}.csv",'w')
+          a
+        end
+      elsif $seperate
         @files = names.inject({}) do |a,name|
-          a[name] = File.open("#{imei}_#{name}.csv",'w')
+          a[name] = File.open("#{$export_prefix}#{imei}_#{name}.csv",'w')
           a
         end
       else
-        @files={nil => File.open("#{imei}.csv",'w')}
+        @files={nil => File.open("#{$export_prefix}#{imei}.csv",'w')}
       end
     end
     @headers = names.inject({}) do |a,name|
@@ -91,7 +162,11 @@ class Export
     end
     @headers[nil] = @headers.values.flatten.sort
     files && files.each do |key,file|
-      file.puts map_headers(base_headers+more_headers+header(key)).join("\t")
+      if $header_maps
+        file.puts $header_maps.find{|hm| hm.event == key}.columns.map{|c| c[0]}.join("\t")
+      else
+        file.puts map_headers(base_headers+more_headers+header(key)).join("\t")
+      end
     end
     if $debug || $verbose
       @headers.each do |name,head|
@@ -108,7 +183,7 @@ class Export
   end
   def more_headers
     $more_headers ?
-    ['IMSI','MSISDN','MCC','MNC','LAC','CI','LAC-CI','RSSI','Platform','Model','OS','Operator'] :
+    ['IMSI','MSISDN','MCC','MNC','LAC','CI','LAC-CI','RSSI','Platform','Model','OS','Operator','Battery'] :
     []
   end
   def base_fields(event)
@@ -127,15 +202,17 @@ class Export
       when 'LAC-CI'
         "#{dataset.recent(event,'service.lac')}-#{dataset.recent(event,'service.cell_id')}"
       when 'MCC'
-        dataset[h] || dataset.recent(event,'service.mcc')
+        event.file[h] || dataset.recent(event,'service.mcc')
       when 'MNC'
-        dataset[h] || dataset.recent(event,'service.mnc')
+        event.file[h] || dataset.recent(event,'service.mnc')
+      when 'Battery'
+        dataset.recent(event,'batteryState.state',600)
       when 'Operator'
-        dataset['carrierName']
+        event.file['carrierName']
       when 'IMSI'
-        dataset.imsi
+        event.file['imsi']
       else
-        dataset[h]
+        event.file[h]
       end
     end
   end
@@ -164,10 +241,10 @@ class Export
     end || hnames
   end
   def export_stats(stats)
-    File.open("#{imei}_stats.csv",'w') do |out|
+    File.open("#{$export_prefix}#{imei}_stats.csv",'w') do |out|
       stats.keys.sort.each do |header|
         out.puts header
-        values = stats[header].keys.sort
+        values = stats[header].keys.sort{|a,b| b.to_s<=>a.to_s}
         out.puts values.join("\t")
         out.puts values.map{|v| stats[header][v]}.join("\t")
         out.puts
@@ -178,7 +255,7 @@ class Export
     @headers[name]
   end
   def puts_to(line,name)
-    name = nil unless($seperate)
+    name = nil unless($seperate || $header_maps)
     files[name].puts(line) if($export && files[name])
   end
   def puts_to_all(line)
@@ -222,13 +299,29 @@ $datasets.keys.sort.each do |imei|
     names = dataset.events_names if(names.length<1)
     export = Export.new(imei,names,dataset)
     export.export_stats(dataset.stats) if($export_stats)
-    events.each do |event|
-      names.each do |name|
-        if event.name === name
-          fields = export.header($seperate ? name : nil).map{|h| event[h]}
-          b_fields = export.base_fields(event) + export.more_fields(event,dataset)
-          export.puts_to "#{b_fields.join("\t")}\t#{fields.join("\t")}", name
-          if_le{puts "#{b_fields.join("\t")}\t#{event.fields.inspect}"}
+    if $header_maps && $header_maps.length > 0
+      $header_maps.each do |hm|
+        puts "Searching for events for header_maps '#{hm.event}'"
+        events.each do |event|
+          if event.name == hm.event
+            header = export.header(event.name)
+            fields = header.map{|h| event[h]}
+            b_header = export.base_headers + export.more_headers
+            b_fields = export.base_fields(event) + export.more_fields(event,dataset)
+            all_fields = hm.map_fields(b_header + header, b_fields + fields)
+            export.puts_to all_fields.join("\t"), event.name
+          end
+        end
+      end
+    else
+      events.each do |event|
+        names.each do |name|
+          if event.name === name
+            fields = export.header($seperate ? name : nil).map{|h| event[h]}
+            b_fields = export.base_fields(event) + export.more_fields(event,dataset)
+            export.puts_to "#{b_fields.join("\t")}\t#{fields.join("\t")}", name
+            if_le{puts "#{b_fields.join("\t")}\t#{event.fields.inspect}"}
+          end
         end
       end
     end

data/examples/csv_chart.rb

@@ -10,7 +10,7 @@ require 'geoptima/options'
 require 'fileutils'
 require 'geoptima/daterange'
 
-Geoptima::assert_version("0.1.3")
+Geoptima::assert_version("0.1.4")
 Geoptima::Chart.available? || puts("No charting libraries available") || exit(-1)
 
 $export_dir = '.'
@@ -26,9 +26,7 @@ $files = Geoptima::Options.process_args do |option|
   option.S {$specfile = ARGV.shift}
   option.P {$diversity = ARGV.shift.to_f}
   option.W {$chart_width = ARGV.shift.to_i}
-  option.T do
-    $time_range = Geoptima::DateRange.from ARGV.shift
-  end
+  option.T {$time_range = Geoptima::DateRange.from ARGV.shift}
 end
 
 FileUtils.mkdir_p $export_dir
@@ -148,8 +146,10 @@ class StatsManager
 end
 
 module Geoptima
+
+  # Class for original stats approach of creating a new 'column' from simple combinations of other columns
   class StatSpec
-    attr_reader :header, :event, :index, :indices, :fields, :options, :proc, :groups
+    attr_reader :header, :event, :index, :indices, :fields, :options, :proc, :groups, :values
     def initialize(header,*fields,&block)
       @header = header
       @fields = fields
@@ -184,13 +184,13 @@ module Geoptima
             key = @group.call(time)
             ghead = "#{header} #{key}"
             @groups[key] = ghead
-            stats_manager.add(map(fields),ghead,nil)
+            stats_manager.add(map_fields(fields),ghead,nil)
           end
         rescue ArgumentError
           puts "Error: Unable to process time field[#{time}]: #{$!}"
         end
       end
-      stats_manager.add(map(fields),header,index)
+      stats_manager.add(map_fields(fields),header,index)
     end
     def div
       unless @div
@@ -224,18 +224,30 @@ module Geoptima
         val
       end
     end
-    def map(values,filter=nil)
+    def prepare_values(values)
+      @values = []
       if @indices
         puts "StatSpec[#{self}]: #{options.inspect}" if($debug)
-        vals = @indices.map{|i| values[i]}
-        puts "\tVALUES:      #{vals.inspect}" if($debug)
-        (options[:filter] || {}).each do |field,expected|
+        @values = @indices.map{|i| values[i]}
+        puts "\tVALUES:      #{values.inspect}" if($debug)
+      end
+      @values
+    end
+    def vals_for(values,filter={})
+      if @indices
+        prepare_values(values)
+        (options[:filter] || filter).each do |field,expected|
           puts "\t\tChecking if field #{field} is #{expected}" if($debug)
           puts "\t\tLooking for #{field} or #{event}.#{field} in #{@fields.inspect}" if($debug)
           hi = @fields.index(field.to_s) || @fields.index("#{event}.#{field}")
-          puts "\t\t#{field} -> #{hi} -> #{hi && vals[hi]}" if($debug)
-          return nil unless(hi && vals[hi] && (expected === vals[hi].downcase || vals[hi].downcase === expected.to_s.downcase))
+          puts "\t\t#{field} -> #{hi} -> #{hi && values[hi]}" if($debug)
+          return nil unless(hi && values[hi] && (expected === values[hi].downcase || values[hi].downcase === expected.to_s.downcase))
         end
+        values
+      end
+    end
+    def map_fields(values,filter={})
+      if vals = vals_for(values,filter)
         val = proc.nil? ? vals[0] : proc.call(*vals)
         puts "\tBLOCK MAP:   #{vals.inspect} --> #{val.inspect}" if($debug)
         if options[:div]
@@ -266,6 +278,128 @@ module Geoptima
       "#{header}[#{index}]<-#{fields.inspect}(#{indices && indices.join(',')})"
     end
   end
+
+  class Group
+    attr_reader :name, :options, :proc, :is_time, :index
+    def initialize(name,options={},&block)
+      @name = name
+      @options = options
+      @proc = block
+      @is_time = options[:is_time]
+    end
+    def index= (ind)
+      puts "Set group header index=#{ind} for group '#{name}'"
+      @index = ind
+    end
+    def call(time,values)
+      is_time && @proc.call(time) || @proc.call(values[index])
+    end
+  end
+
+  # The KPI class allows for complex statistics called 'Key Performance Indicators'.
+  # These are specified using four functions:
+  # filter: how to choose rows to include in the statistics (default is '!map.nil?')
+  # map: how to convert a row into the internal stats (default is input columns)
+  # aggregate: how to aggregate internal stats to higher levels (eg. daily, default is count)
+  # reduce: how to extract presentable stats from internal stats (eg. avg=total/count, default is internal stats)
+  #
+  # The KPI is defined with a name and set of columns to use, followed by the block
+  # defining the four functions above. For example:
+  #
+  # kpi 'DNS Success', 'dnsLookup.address', 'dnsLookup.error', 'dnsLookup.interface' do |f|
+  #   f.filter {|addr,err,int| addr =~/\w/}
+  #   f.map {|addr,err,int| err.length==0 ? [1,1] : [1,0]}
+  #   f.aggregate {|a,v| a[0]+=v[0];a[1]+=v[1];a}
+  #   f.reduce {|a| 100.0*a[1].to_f/a[0].to_f}
+  # end
+  #
+  # Currently this class extends StatSpec for access to the prepare_indices method.
+  # We should consider moving that to a mixin, or depreciating the StatSpec class
+  # entirely since KPISpec should provide a superset of features.
+  class KPISpec < StatSpec
+    def initialize(header,*fields,&block)
+      @header = header
+      @fields = fields
+      @event = @fields[0].split(/\./)[0]
+      block.call self unless(block.nil?)
+      if @fields[-1].is_a?(Hash)
+        @options = @fields.pop
+      else
+        @options = {}
+      end
+      @group_procs = []
+      @groups = {}
+      if @options[:group]
+        [@options[:group]].flatten.compact.sort.uniq.each do |group_name|
+          gname = group_name.to_s.intern
+          case gname
+          when :months
+            group_by(gname,true) {|t| t.strftime("%Y-%m")}
+          when :weeks
+            group_by(gname,true) {|t| t.strftime("%Y w%W")}
+          when :days
+            group_by(gname,true) {|t| t.strftime("%Y-%m-%d")}
+          when :hours
+            group_by(gname,true) {|t| t.strftime("%Y-%m-%d %H")}
+          else
+            group_by(gname) {|f| f}
+          end
+        end
+      end
+      puts "Created StatSpec: #{self}"
+    end
+    def group_by(field,is_time=false,&block)
+      @group_procs = Group.new(field,:is_time => is_time,&block)
+    end
+    def filter(&block)
+      @filter_proc = block
+    end
+    def map(&block)
+      @map_proc = block
+    end
+    def aggregate(&block)
+      @aggregate_proc = block
+    end
+    def reduce(&block)
+      @reduce_proc = block
+    end
+    def add(stats_manager,values)
+      prepare_values(values)
+      if @group_procs.length > 0
+        begin
+          time = DateTime.parse(values[stats_manager.time_index])
+          if $time_range.nil? || $time_range.include?(time)
+            key = @group_procs.inject(header) do |ghead,group|
+              key = @group.call(time,values)
+              ghead += " #{key}"
+            end
+            @groups[key] = ghead
+            stats_manager.add(map_fields(fields),ghead,nil)
+          end
+        rescue ArgumentError
+          puts "Error: Unable to process time field[#{time}]: #{$!}"
+        end
+      end
+      stats_manager.add(map_fields(fields),header,index)
+    end
+    def map_fields(values,filter=nil)
+      if values
+        if @filter_proc.nil? || @filter_proc.call(*values)
+          val = @map_proc && @map_proc.call(*values) || values[0]
+          puts "\tBLOCK MAP:   #{values.inspect} --> #{values.inspect}" if($debug)
+        end
+        val
+      end
+    end
+    def prepare_indices(stats_manager,headers)
+      super(stats_manager,headers)
+      @group_procs.each do |g|
+        g.index = fields.index(g.name)
+      end
+    end
+  end
+
+  # Class for specifications of individual charts
   class ChartSpec
     attr_reader :chart_type, :header, :options
     def initialize(header,options={})
@@ -328,14 +462,15 @@ module Geoptima
       g.write("#{$export_dir}/Chart_#{stats_manager.name}_#{header}_#{chart_type}_distribution.png")
     end
     def to_s
-      "#{chart_type.upcase}-#{header}"
+      "#{chart_type.to_s.upcase}-#{header}"
     end
   end
   class StatsSpecs
-    attr_reader :chart_specs, :stat_specs
+    attr_reader :chart_specs, :stat_specs, :kpi_specs
     def initialize(specfile)
       @chart_specs = []
       @stat_specs = []
+      @kpi_specs = []
       instance_eval(File.open(specfile).read)
     end
     def category_chart(header,options={})
@@ -353,10 +488,16 @@ module Geoptima
     def stats(header,*fields,&block)
       @stat_specs << StatSpec.new(header,*fields,&block)
     end
+    def kpi(header,*fields,&block)
+      @kpi_specs << KPISpec.new(header,*fields,&block)
+    end
     def add_stats(stats_manager,headers)
       stat_specs.each do |stat_spec|
         stat_spec.prepare_indices(stats_manager,headers)
       end
+      kpi_specs.each do |kpi_spec|
+        kpi_spec.prepare_indices(stats_manager,headers)
+      end
     end
     def add_fields(stats_manager,fields)
       puts "Adding fields to #{stat_specs.length} StatSpec's" if($debug)
@@ -364,9 +505,14 @@ module Geoptima
         puts "Adding fields to StatSpec: #{stat_spec}" if($debug)
         stat_spec.add(stats_manager,fields)
       end
+      puts "Adding fields to #{kpi_specs.length} KPISpec's" if($debug)
+      kpi_specs.each do |kpi_spec|
+        puts "Adding fields to KPISpec: #{kpi_spec}" if($debug)
+        kpi_spec.add(stats_manager,fields)
+      end
     end
     def to_s
-      "Stats[#{@stat_specs.join(', ')}] AND Charts[#{@chart_specs.join(', ')}]"
+      "Stats[#{@stat_specs.join(', ')}] AND KPIs[#{@kpi_specs.join(', ')}] AND Charts[#{@chart_specs.join(', ')}]"
     end
   end
 end
@@ -468,7 +614,11 @@ end
 $stats_managers.each do |name,stats_manager|
   if $specs
     $specs.chart_specs.each do |chart_spec|
-      chart_spec.process(stats_manager)
+      begin
+        chart_spec.process(stats_manager)
+      rescue NoMethodError
+        puts "Failed to process chart '#{chart_spec}': #{$!}"
+      end
     end
   end
   if $create_all