geoptima 0.1.3 → 0.1.4

data/examples/csv_merge.rb

@@ -9,20 +9,24 @@ require 'geoptima/options'
 require 'fileutils'
 require 'geoptima/daterange'
 
-Geoptima::assert_version("0.1.3")
+Geoptima::assert_version("0.1.4")
 
 $export_dir = '.'
 $export_name = 'merged.csv'
+$split_by = :days
 
 $files = Geoptima::Options.process_args do |option|
   option.t {$time_split = true}
+  option.m {$low_memory = true}
   option.D {$export_dir = ARGV.shift}
   option.N {$export_name = ARGV.shift}
-  option.T do
-    $time_range = Geoptima::DateRange.new(*(ARGV.shift.split(/[\,]+/).map do |t|
-      DateTime.parse t
-    end))
+  option.S do
+    $split_by = case ARGV.shift.downcase.intern
+    when :days ; :days
+    else :days
+    end
   end
+  option.T {$time_range = Geoptima::DateRange.from ARGV.shift}
 end
 
 FileUtils.mkdir_p $export_dir
@@ -30,46 +34,216 @@ FileUtils.mkdir_p $export_dir
 $help = true unless($files.length>0)
 if $help
   puts <<EOHELP
-Usage: csv_chart <-dht> <-N name> <-D dir> <-T range> files...
+Usage: csv_merge <-dhtm> <-N name> <-D dir> <-T range> <-S split_by> files...
  -d  debug mode #{cw $debug}
  -h  print this help #{cw $help}
- -t  merge and split by time (days) #{cw $time_split}
- -N  use specified name for merged dataset: #{$merged_name}
+ -t  merge and split by time (#{$split_by}) #{cw $time_split}
+ -m  use low memory, temporarily storing to intermediate files #{cw $low_memory}
+ -N  use specified name for merged dataset: #{$export_name}
  -D  export to specified directory: #{$export_dir}
+ -S  time units to split exports by: #{$split_by}
  -T  set time-range filter: #{$time_range}
 Files to import: #{$files.join(', ')}
 EOHELP
   exit
 end
 
-class CSVData
-  attr_reader :headers
-  def initialize(headers)
+class CSVRecord
+  attr_reader :time, :fields, :day
+  def initialize(fields,time_index=0)
+    @fields = fields
+    @time = DateTime.parse(fields[time_index])
+    @day = @time.strftime("%Y-%m-%d")
+  end
+  def [](index)
+    fields[index]
+  end
+  def <=>(other)
+    time <=> other
+  end
+  def within(time_range)
+    time_range.nil? || time_range.include?(time)
   end
 end
 
-$files.each do |file|
-  lines = 0
-  headers = nil
-  filename = File.basename(file)
-  (names = filename.split(/[_\.]/)).pop
-  name = $merge_all ? ($merged_name || 'All') : names.join('_')
-  $stats_managers[name] ||= StatsManager.new(name)
-  puts "About to read file #{file}"
-  File.open(file).each do |line|
-    lines += 1
-    fields=line.chomp.split(/\t/)
-    if headers
-      puts "Processing line: #{line}" if($debug)
-      $stats_managers[name].add_all(fields,headers)
+class CSVDataset
+  attr_reader :filename, :headers, :day_map, :lines, :count, :record_creation_duration
+  def initialize(filename)
+    @filename = filename
+    @lines = []
+    @day_map = {}
+    @record_creation_duration = 0
+    @count = 0
+    @headers = nil
+    read_file do |fields|
+      add fields
+    end
+  end
+  def read_file
+    lines = 0
+    File.open(filename).each do |line|
+      fields=line.chomp.split(/\t/)
+      if lines > 0
+        puts "Processing line: #{line}" if($debug)
+        yield fields
+      else
+        if fields.length<2
+          puts "Too few headers, rejecting #{file}"
+          break
+        end
+        @headers ||= fields
+      end
+      lines += 1
+    end
+    @export_headers ||= @headers
+  end
+  def add(fields)
+    start_time = Time.new
+    line = create_line(fields)
+    if line
+      @day_map[line.day] ||= 0
+      @day_map[line.day] += 1
+      @lines << line unless($low_memory)
+      @count += 1
+      @record_creation_duration += Time.new - start_time
+    end
+    line
+  end
+  def create_line(fields)
+    begin
+      line = CSVRecord.new(fields,0)
+      if(line.within($time_range))
+        line
+      else
+        nil
+      end
+    rescue ArgumentError
+      puts "Failed to parse line with timestamp='#{fields[0]}': #{$!}"
+    end
+  end
+  def header_map(eh=nil)
+    if eh
+      @export_headers = eh
+      @header_map = nil
+    end
+    unless @header_map
+      @header_map = []
+      (@export_headers || @headers).each do |head|
+        @header_map << @headers.index(head)
+      end
+    end
+    @header_map
+  end
+  def map_line(line)
+    @header_map.map do |index|
+      index && line[index]
+    end
+  end
+  def days
+    @day_map.keys.sort
+  end
+  def each(eh=nil)
+    header_map(eh)
+    if $low_memory
+      read_file do |fields|
+        line = create_line fields
+        yield line.day,map_line(line)
+      end
     else
-      headers = fields
-      if headers.length<2
-        puts "Too few headers, rejecting #{file}"
-        break
+      (@lines || []).each do |line|
+        yield line.day,map_line(line)
       end
-      $stats_managers[name].set_headers(headers)
     end
   end
+  def <=>(other)
+    self.filename <=> other.filename
+  end
+  def length
+    count
+  end
 end
 
+class CSVDatasets
+  attr_reader :datasets
+  def initialize
+    @datasets = []
+  end
+  def add_file(file)
+    lines = 0
+    dataset = nil
+    filename = File.basename(file)
+    (names = filename.split(/[_\.]/)).pop
+    name = names.join('_')
+    puts "About to read file #{file}"
+    dataset = CSVDataset.new(file)
+    @datasets << dataset if(dataset && dataset.length>0)
+    dataset
+  end
+  def export_days
+    headers = @datasets.map{|d| d.headers}.flatten.uniq
+    days = @datasets.map{|d| d.days}.flatten.sort.uniq
+    day_files = {}
+    day_names = {}
+    count = {}
+    duration = {}
+    days.each do |day|
+      filename = "#{$export_dir}/#{$export_name.gsub(/\.csv$/,'')}_#{day}.csv"
+      puts "Exporting #{filename} for #{day}"
+      day_names[day] = filename
+      day_files[day] = File.open(filename,'w')
+      day_files[day].puts headers.join("\t")
+      count[day] = 0
+      duration[day] = 0
+    end
+    @datasets.sort.each do |dataset|
+      dataset.each(headers) do |day,line|
+        start_time = Time.new
+        day_files[day].puts line.join("\t")
+        duration[day] += Time.new - start_time
+        count[day] += 1
+      end
+    end
+    day_files.each do |day,out|
+      out.close
+      puts "\tExported #{count[day]} records to #{day_names[day]} in #{duration[day]} seconds"
+    end
+  end
+  def export_merged
+    headers = @datasets.map{|d| d.headers}.flatten.sort.uniq
+    filename = "#{$export_dir}/#{$export_name}"
+    File.open(filename,'w') do |out|
+      out.puts headers.join("\t")
+      @datasets.sort.each(headers) do |dataset|
+        dataset.each do |day,line|
+          out.puts line.join("\t")
+        end
+      end
+    end
+  end
+end
+
+$datasets = CSVDatasets.new
+
+$files.each do |file|
+  start_time = Time.new
+  ds = $datasets.add_file(file)
+  duration = Time.new - start_time
+  puts "\tLoaded #{file} in #{duration} seconds"
+  puts "\t#{(100.0 * ds.record_creation_duration.to_f/duration.to_f).to_i}% = #{ds.record_creation_duration}/#{duration} was spent creating records"
+  puts "\tFile contained #{ds.length} events for #{ds.days.length} days:"
+  ds.days.each do |day|
+    puts "\t\t#{day}: \t#{(100.0 * ds.day_map[day].to_f/ds.length.to_f).to_i}%\t#{ds.day_map[day]} records"
+  end
+end
+
+start_time = Time.new
+
+if $time_split
+  $datasets.export_days
+else
+  $datasets.export_merged
+end
+
+duration = Time.new - start_time
+puts "Exported in #{duration} seconds"
+

data/examples/show_geoptima.rb

@@ -6,8 +6,9 @@ $: << '../lib'
 
 require 'date'
 require 'geoptima'
+require 'geoptima/options'
 
-Geoptima::assert_version("0.1.3")
+Geoptima::assert_version("0.1.4")
 
 $debug=false
 
@@ -24,29 +25,89 @@ $files = Geoptima::Options.process_args do |option|
   option.m {$map_headers = true}
   option.a {$combine_all = true}
   option.l {$more_headers = true}
+  option.P {$export_prefix = ARGV.shift}
   option.E {$event_names += ARGV.shift.split(/[\,\;\:\.]+/)}
-  option.T do
-    $time_range = Geoptima::DateRange.from ARGV.shift
-  end
+  option.T {$time_range = Geoptima::DateRange.from ARGV.shift}
   option.L {$print_limit = [1,ARGV.shift.to_i].max}
-
-  option.t {$time_split = true}
-  option.D {$export_dir = ARGV.shift}
-  option.N {$merged_name = ARGV.shift}
-  option.S {$specfile = ARGV.shift}
-  option.P {$diversity = ARGV.shift.to_f}
-  option.W {$chart_width = ARGV.shift.to_i}
-  option.T do
-    $time_range = Geoptima::DateRange.from ARGV.shift
-  end
+  option.M {$mapfile = ARGV.shift}
 end.map do |file|
   File.exist?(file) ? file : puts("No such file: #{file}")
 end.compact
 
+class HeaderMap
+  attr_reader :prefix, :name, :event
+  attr_accessor :columns
+  def initialize(prefix,name,event)
+    @prefix = prefix
+    @name = name
+    @event = event
+    @columns = []
+  end
+  def mk_known(header)
+    puts "Creating column mappings for headers: #{header}" if($debug)
+    @col_indices = {}
+    columns.each do |col|
+      c = (col[1] && col[1].gsub(/\?/,'')).to_s
+      if c.length>0
+        @col_indices[c] = header.index(c)
+        puts "\tMade column mapping: #{c} --> #{header.index(c)}" if($debug)
+      end
+    end
+  end
+  def map_fields(header,fields)
+    @scenario_counter ||= 0
+    mk_known(header) unless @col_indices
+    @columns.map do |column|
+      if column[1] =~ /SCENARIO_COUNTER/
+        @scenario_counter += 1
+      else
+        index = @col_indices[column[1]]
+        puts "Found mapping #{column} -> #{index} -> #{index && fields[index]}" if($debug)
+        index && fields[index]
+      end
+    end
+  end
+end
+
+if $mapfile
+  $header_maps = []
+  current_map = nil
+  prefix = $mapfile.split(/\./)[0]
+  File.open($mapfile).each do |line|
+    line.chomp!
+    next if line =~ /^\s*#/
+    next if line.length < 2
+    if line =~ /^\[(\w+)\]\t(\w+)/
+      current_map = HeaderMap.new(prefix,$1,$2)
+      $header_maps << current_map
+    elsif current_map
+      current_map.columns << line.chomp.split(/\t/)[0..1]
+    else
+      puts "Invalid header map line: #{line}"
+    end
+  end
+end
+
+def show_header_maps
+  if $header_maps
+    puts "Using #{$header_maps.length} header maps:"
+    $header_maps.each do |hm|
+      puts "\t[#{hm.name}] (#{hm.event})"
+      if $debug
+        hm.columns.each do |hc|
+          puts "\t\t#{hc.map{|c| (c+' '*30)[0..30]}.join("\t-->\t")}"
+        end
+      else
+        puts "\t\t#{hm.columns.map{|hc| hc[0]}.join(', ')}"
+      end
+    end
+  end
+end
+
 $help = true if($files.length < 1)
 if $help
   puts <<EOHELP
-Usage: show_geoptima <-dpvxomlsah> <-L limit> <-E types> <-T min,max> file <files>
+Usage: show_geoptima <-dpvxomlsah> <-L limit> <-E types> <-T min,max> <-M mapfile> file <files>
   -d  debug mode (output more context during processing) #{cw $debug}
   -p  print mode (print out final results to console) #{cw $print}
   -v  verbose mode (output extra information to console) #{cw $verbose}
@@ -57,14 +118,19 @@ Usage: show_geoptima <-dpvxomlsah> <-L limit> <-E types> <-T min,max> file <file
   -s  seperate the export files by event type #{cw $seperate}
   -a  combine all IMEI's into a single dataset #{cw $combine_all}
   -h  show this help
+  -P  prefix for exported files (default: ''; current: #{$export_prefix})
   -E  comma-seperated list of event types to show and export (default: all; current: #{$event_names.join(',')})
   -T  time range to limit results to (default: all; current: #{$time_range})
   -L  limit verbose output to specific number of lines #{cw $print_limit}
+  -M  mapfile of normal->altered header names: #{$mapfile}
 EOHELP
+  show_header_maps
   exit 0
 end
 
 $verbose = $verbose || $debug
+show_header_maps if($verbose)
+
 $datasets = Geoptima::Dataset.make_datasets($files, :locate => true, :time_range => $time_range, :combine_all => $combine_all)
 
 class Export
@@ -74,13 +140,18 @@ class Export
     @imei = imei
     @names = names
     if $export
-      if $seperate
+      if $header_maps
+        @files = $header_maps.inject({}) do |a,hm|
+          a[hm.event] = File.open("#{$export_prefix}#{imei}_#{hm.prefix}_#{hm.name}.csv",'w')
+          a
+        end
+      elsif $seperate
         @files = names.inject({}) do |a,name|
-          a[name] = File.open("#{imei}_#{name}.csv",'w')
+          a[name] = File.open("#{$export_prefix}#{imei}_#{name}.csv",'w')
           a
         end
       else
-        @files={nil => File.open("#{imei}.csv",'w')}
+        @files={nil => File.open("#{$export_prefix}#{imei}.csv",'w')}
       end
     end
     @headers = names.inject({}) do |a,name|
@@ -91,7 +162,11 @@ class Export
     end
     @headers[nil] = @headers.values.flatten.sort
     files && files.each do |key,file|
-      file.puts map_headers(base_headers+more_headers+header(key)).join("\t")
+      if $header_maps
+        file.puts $header_maps.find{|hm| hm.event == key}.columns.map{|c| c[0]}.join("\t")
+      else
+        file.puts map_headers(base_headers+more_headers+header(key)).join("\t")
+      end
     end
     if $debug || $verbose
       @headers.each do |name,head|
@@ -108,7 +183,7 @@ class Export
   end
   def more_headers
     $more_headers ?
-    ['IMSI','MSISDN','MCC','MNC','LAC','CI','LAC-CI','RSSI','Platform','Model','OS','Operator'] :
+    ['IMSI','MSISDN','MCC','MNC','LAC','CI','LAC-CI','RSSI','Platform','Model','OS','Operator','Battery'] :
     []
   end
   def base_fields(event)
@@ -127,15 +202,17 @@ class Export
       when 'LAC-CI'
         "#{dataset.recent(event,'service.lac')}-#{dataset.recent(event,'service.cell_id')}"
       when 'MCC'
-        dataset[h] || dataset.recent(event,'service.mcc')
+        event.file[h] || dataset.recent(event,'service.mcc')
       when 'MNC'
-        dataset[h] || dataset.recent(event,'service.mnc')
+        event.file[h] || dataset.recent(event,'service.mnc')
+      when 'Battery'
+        dataset.recent(event,'batteryState.state',600)
       when 'Operator'
-        dataset['carrierName']
+        event.file['carrierName']
       when 'IMSI'
-        dataset.imsi
+        event.file['imsi']
       else
-        dataset[h]
+        event.file[h]
       end
     end
   end
@@ -164,10 +241,10 @@ class Export
     end || hnames
   end
   def export_stats(stats)
-    File.open("#{imei}_stats.csv",'w') do |out|
+    File.open("#{$export_prefix}#{imei}_stats.csv",'w') do |out|
       stats.keys.sort.each do |header|
         out.puts header
-        values = stats[header].keys.sort
+        values = stats[header].keys.sort{|a,b| b.to_s<=>a.to_s}
         out.puts values.join("\t")
         out.puts values.map{|v| stats[header][v]}.join("\t")
         out.puts
@@ -178,7 +255,7 @@ class Export
     @headers[name]
   end
   def puts_to(line,name)
-    name = nil unless($seperate)
+    name = nil unless($seperate || $header_maps)
     files[name].puts(line) if($export && files[name])
   end
   def puts_to_all(line)
@@ -222,13 +299,29 @@ $datasets.keys.sort.each do |imei|
     names = dataset.events_names if(names.length<1)
     export = Export.new(imei,names,dataset)
     export.export_stats(dataset.stats) if($export_stats)
-    events.each do |event|
-      names.each do |name|
-        if event.name === name
-          fields = export.header($seperate ? name : nil).map{|h| event[h]}
-          b_fields = export.base_fields(event) + export.more_fields(event,dataset)
-          export.puts_to "#{b_fields.join("\t")}\t#{fields.join("\t")}", name
-          if_le{puts "#{b_fields.join("\t")}\t#{event.fields.inspect}"}
+    if $header_maps && $header_maps.length > 0
+      $header_maps.each do |hm|
+        puts "Searching for events for header_maps '#{hm.event}'"
+        events.each do |event|
+          if event.name == hm.event
+            header = export.header(event.name)
+            fields = header.map{|h| event[h]}
+            b_header = export.base_headers + export.more_headers
+            b_fields = export.base_fields(event) + export.more_fields(event,dataset)
+            all_fields = hm.map_fields(b_header + header, b_fields + fields)
+            export.puts_to all_fields.join("\t"), event.name
+          end
+        end
+      end
+    else
+      events.each do |event|
+        names.each do |name|
+          if event.name === name
+            fields = export.header($seperate ? name : nil).map{|h| event[h]}
+            b_fields = export.base_fields(event) + export.more_fields(event,dataset)
+            export.puts_to "#{b_fields.join("\t")}\t#{fields.join("\t")}", name
+            if_le{puts "#{b_fields.join("\t")}\t#{event.fields.inspect}"}
+          end
         end
       end
     end

data/geoptima.gemspec

@@ -25,7 +25,7 @@ EOF
   s.files        = Dir.glob("{bin,lib,rdoc}/**/*").reject{|x| x=~/(tmp|target|test-data)/ || x=~/~$/} +
                    Dir.glob("examples/*rb") + Dir.glob("examples/sample*json") + 
                    %w(README.rdoc CHANGELOG CONTRIBUTORS Gemfile geoptima.gemspec)
-  s.executables  = ['show_geoptima','geoptima_file_time','csv_chart','csv_stats']
+  s.executables  = ['show_geoptima','geoptima_file_time','csv_chart','csv_stats','csv_merge']
 
   s.extra_rdoc_files = %w( README.rdoc )
   s.rdoc_options = ["--quiet", "--title", "Geoptima.rb", "--line-numbers", "--main", "README.rdoc", "--inline-source"]

data/lib/geoptima/data.rb

@@ -55,7 +55,7 @@ module Geoptima
       @header = header
       @data = data
       @fields = @header.inject({}) do |a,v|
-        a[v] = @data[a.length]
+        a[v] = check_field(@data[a.length])
         a
       end
       @time = start + (@fields['timeoffset'].to_f / MSPERDAY.to_f)
@@ -65,6 +65,9 @@ module Geoptima
       end
       puts "Created Event: #{self}" if($debug)
     end
+    def check_field(field)
+      (field && field.respond_to?('length') && field =~ /\d\,\d/) ? field.gsub(/\,/,'.').to_f : field
+    end
     def utc
       time.new_offset(0)
     end
@@ -135,7 +138,7 @@ module Geoptima
       @start ||= subscriber['start'] && DateTime.parse(subscriber['start'].gsub(/Asia\/Bangkok/,'GMT+7'))#.gsub(/Mar 17 2044/,'Feb 14 2012'))
     end
     def valid?
-      start && start > Data.min_start && start < Data.max_start
+      start && start >= (Data.min_start-1) && start < Data.max_start
     end
     def self.min_start
       @@min_start ||= MIN_DATETIME
@@ -180,30 +183,28 @@ module Geoptima
         unless header
           puts "No header found for '#{event_type}', trying known Geoptima headers"
           header = Event::KNOWN_HEADERS[event_type]
-          if header
-            puts "Found known header '#{event_type}' => #{header.inspect}"
-            if data = events_data[event_type]
-              mismatch = data.length % header.length
-              if mismatch != 0
-                puts "Known header length #{header.length} incompatible with data length #{events_data[event_type].length}"
-                header = nil
-              end
-            else
-              puts "No data found for event type '#{event_type}'"
-              header = nil
-            end
+          puts "Found known header '#{event_type}' => #{header.inspect}" if(header)
+        end
+        # Double-check the header length matches a multiple of the data length
+        if header
+          mismatch = events.length % header.length
+          if mismatch != 0
+            puts "'#{event_type}' header length #{header.length} incompatible with data length #{events.length}"
+            header = nil
           end
+        else
+          puts "No header found for event type: #{event_type}"
         end
         # Now process the single long data array into a list of events with timestamps
         if header
           events_data[event_type] = (0...data[event_type].to_i).inject([]) do |a,block|
             index = header.length * block
-            data = events[index...(index+header.length)]
-            if data && data.length == header.length
+            record = events[index...(index+header.length)]
+            if record && record.length == header.length
               @count += 1
-              a << Event.new(self,start,event_type,header,data)
+              a << Event.new(self,start,event_type,header,record)
             else
-              puts "Invalid '#{event_type}' data block #{block}: #{data.inspect}"
+              puts "Invalid '#{event_type}' data block #{block}: #{record.inspect}"
               break a
             end
           end
@@ -214,8 +215,6 @@ module Geoptima
               puts "\t#{d.data.join("\t")}"
             end
           end
-        else
-          puts "No header found for event type: #{event_type}"
         end
       end
       find_first_and_last(events_data)
@@ -227,8 +226,8 @@ module Geoptima
       events_data.each do |event_type,data|
         @first ||= data[0]
         @last ||= data[-1]
-        @first = data[0] if(@first.time > data[0].time)
-        @last = data[-1] if(@last.time < data[-1].time)
+        @first = data[0] if(@first && @first.time > data[0].time)
+        @last = data[-1] if(@last && @last.time < data[-1].time)
       end
       if $debug
         puts "For data: #{self}"
@@ -276,24 +275,28 @@ module Geoptima
       end.compact.uniq
     end
 
-    def recent(event,key)
+    def recent(event,key,seconds=60)
       unless event[key]
-        puts "Searching for recent values for '#{key}' starting at event #{event}" if($debug)
-        ev,prop=key.split(/\./)
-        ar=sorted
-        puts "\tSearching through #{ar && ar.length} events for event type #{ev} and property #{prop}" if($debug)
-        if i=ar.index(event)
-          afe = while(i>0)
-            fe = ar[i-=1]
-            puts "\t\tTesting event[#{i}]: #{fe}" if($debug)
-            break(fe) if(fe.nil? || fe.name == ev || (event.time - fe.time) * SPERDAY > 60)
-          end
-          if afe && afe.name == ev
-            puts "\t\tFound event[#{i}] with #{prop} => #{afe[prop]} and time gap of #{(event.time - fe.time) * SPERDAY} seconds" if($verbose)
-            event[key] = afe[prop]
+        if imei = event.file.imei
+          puts "Searching for recent values for '#{key}' starting at event #{event}" if($debug)
+          ev,prop=key.split(/\./)
+          ar=sorted
+          puts "\tSearching through #{ar && ar.length} events for event type #{ev} and property #{prop}" if($debug)
+          if i=ar.index(event)
+            afe = while(i>0)
+              fe = ar[i-=1]
+              puts "\t\tTesting event[#{i}]: #{fe}" if($debug)
+              break(fe) if(fe.nil? || (event.time - fe.time) * SPERDAY > seconds || (fe.name == ev && fe.file.imei == imei))
+            end
+            if afe && afe.name == ev
+              puts "\t\tFound event[#{i}] with #{prop} => #{afe[prop]} and time gap of #{(event.time - fe.time) * SPERDAY} seconds" if($debug)
+              event[key] = afe[prop]
+            end
+          else
+            puts "Event not found in search for recent '#{key}': #{event}"
           end
         else
-          puts "Event not found in search for recent '#{key}': #{event}"
+          puts "Not searching for correlated data without imei: #{event}"
         end
       end
 #      @recent[key] ||= ''

data/lib/geoptima/version.rb

@@ -1,6 +1,6 @@
 module Geoptima
 
-  VERSION = "0.1.3"
+  VERSION = "0.1.4"
 
   def self.version_as_int(ver)
     base = 1