geoptima 0.0.6 → 0.0.7

data/bin/csv_chart

@@ -0,0 +1,132 @@
+#!/usr/bin/ruby
+
+# useful if being run inside a source code checkout
+$: << 'lib'
+$: << '../lib'
+
+require 'geoptima/chart'
+require 'geoptima/options'
+
+$files = Geoptima::Options.process_args
+
+class Stats
+  attr_reader :file, :imei, :headers, :stats, :data
+  def initialize(file,imei,fields)
+    @file = file
+    @imei = imei
+    @headers = fields
+    @stats = fields.map{|h| {}}
+    @data = fields.map{|h| []}
+    @numerical = fields.map{|h| true}
+  end
+  def add(field,index)
+    if field =~ /\w/
+      @numerical[index] &&= is_number?(field)
+      puts "\tField[#{index}]: #{field}" if($debug)
+      stats = @stats[index]
+      stats[field] ||= 0
+      stats[field] += 1
+      puts "\tField[#{index}]: #{field} => #{stats[field]}" if($debug)
+      @data[index] << field
+    end
+  end
+  def is_number?(field)
+    is_integer?(field) || is_float?(field)
+  end
+  def is_integer?(field)
+    field.to_i.to_s == field
+  end
+  def is_float?(field)
+    field.to_f.to_s == field
+  end
+  def length(index)
+    @stats[index].length
+  end
+  def diversity(index)
+    100.0 * @stats[index].length.to_f / @data[index].length.to_f
+  end
+  def diverse?(index)
+    @stats[index].length>500 || diversity(index) > 40.0
+  end
+  def numerical?(index)
+    @numerical[index]
+  end
+end
+
+$stats = {}
+
+$files.each do |file|
+  lines = 0
+  imei = file.split(/[_\.]/)[0]
+  File.open(file).each do |line|
+    lines += 1
+    fields=line.chomp.split(/\t/)
+    if $stats[file]
+      puts "Processing line: #{line}" if($debug)
+      fields.each_with_index do |field,index|
+        $stats[file].add(field,index)
+      end
+    else
+      $stats[file] = Stats.new(file,imei,fields)
+    end
+  end
+end
+
+$stats.each do |file,stats|
+  stats.headers.each_with_index do |header,index|
+    puts "Charting #{header} with diversity #{stats.diversity(index)}"
+    case header
+    when 'signal.strength'
+      Geoptima::Chart.draw_line_chart(
+        stats.imei,
+        stats.data[0],
+        stats.data[index].map{|f| v=f.to_i; (v>-130 && v<0) ? v : nil},
+        :title => 'Signal Strength',
+        :maximum_value => -30,
+        :minimum_value => -130,
+        :width => 1024
+      ).write("Chart_#{stats.imei}_#{header}.png")
+
+      hist = stats.stats[index]
+      keys = hist.keys.sort{|a,b| a.to_i <=> b.to_i}
+      values = keys.map{|k| hist[k]}
+      Geoptima::Chart.draw_histogram_chart(
+        stats.imei, keys, values,
+        :title => 'Signal Strength Distribution',
+        :width => 1024
+      ).write("Chart_#{stats.imei}_#{header}_distribution.png")
+
+    when 'Event'
+      hist = stats.stats[index]
+      keys = hist.keys.sort{|a,b| a.to_i <=> b.to_i}
+      values = keys.map{|k| hist[k]}
+      Geoptima::Chart.draw_category_chart(
+        stats.imei, keys, values,
+        :title => "#{header} Distribution",
+        :width => 1024
+      ).write("Chart_#{stats.imei}_#{header}_distribution.png")
+
+    else
+      if stats.diverse?(index)
+        puts "Ingnoring high diversity field #{header}"
+      else
+        puts "Charting field: #{header} with length #{stats.length(index)} and diversity #{stats.diversity(index)}"
+        hist = stats.stats[index]
+        keys = hist.keys.sort{|a,b| a.to_i <=> b.to_i}
+        values = keys.map{|k| hist[k]}
+        args = [stats.imei, keys, values, {
+          :title => "#{header} Distribution",
+          :width => 1024}]
+        g = (stats.length(index) > 50) ?
+              Geoptima::Chart.draw_line_chart(*args) :
+            (stats.length(index) > 10 || stats.numerical?(index)) ?
+              Geoptima::Chart.draw_histogram_chart(*args) :
+            (stats.length(index) > 1) ?
+              Geoptima::Chart.draw_category_chart(*args) :
+            nil
+        g && g.write("Chart_#{stats.imei}_#{header}_distribution.png")
+      end
+    end
+  end
+end
+

data/bin/csv_stats

@@ -0,0 +1,126 @@
+#!/usr/bin/env ruby
+
+# useful if being run inside a source code checkout
+$: << 'lib'
+$: << '../lib'
+
+require 'geoptima/options'
+
+$show_all = false
+
+$files = Geoptima::Options.process_args do |option|
+  option.a {$show_all = true}
+  option.o {$output_files = true}
+  option.u {$exclude_stats = true}
+  option.I {$include_props = ARGV.shift.split(/[\,\;]+/)}
+  option.O {$output_prefix = ARGV.shift}
+end
+
+$help = true if($files.length<0)
+if $help
+  puts <<EOHELP
+Usage: ./stats.rb <-dha> <-I include_props> files
+ -d  debug mode #{cw $debug}
+ -h  print this help #{cw $help}
+ -a  show all values, not just discrete ones #{cw $show_all}
+ -o  output the stats to files instead of the console #{cw $output_files}
+ -u  exclude stats, showing only unique lists of values #{cw $exclude_stats}
+ -I  only calculate statistics for named properties #{aw $include_props}
+ -O  prefix output files with specific prefix #{aw $output_prefix}
+EOHELP
+  exit 0
+end
+
+$files.each do |file|
+  lines = 0
+  headers = nil
+  file_stats = nil
+  File.open(file).each do |line|
+    lines += 1
+    fields=line.chomp.split(/[\t\,]/)
+    if headers
+      puts "Processing line: #{line}" if($debug)
+      lac,ci=nil
+      fields.each_with_index do |field,index|
+        if $include_props.to_s=='' || $include_props.index(headers[index])
+          puts "\tField[#{index}]: #{field}" if($debug)
+          stats = file_stats[index]
+          stats[field] ||= 0
+          stats[field] += 1
+          puts "\tField[#{index}]: #{field} => #{stats[field]}" if($debug)
+          lac=field if(headers[index]=='LAC' || headers[index]=='service.lac')
+          ci=field if(headers[index]=='CI' || headers[index]=='service.cell_id')
+          #puts "\tSet LAC=#{lac}, CI=#{ci} based on header #{headers[index]}" if($debug)
+        end
+      end
+      puts "\tSet LAC=#{lac}, CI=#{ci}" if($debug)
+      if lac && ci
+        index = headers.length - 1
+        puts "\tAdding statistics for LAC=#{lac}, CI=#{ci} using additional header '#{headers[index]}'" if($debug)
+        stats = file_stats[index]
+        field="#{lac}-#{ci}"
+        stats[field] ||= 0
+        stats[field] += 1
+        puts "\tField[#{index}]: #{field} => #{stats[field]}" if($debug)
+      end
+      if $debug
+        headers.each_with_index do |header,index|
+          stats = file_stats[index]
+          values = stats.keys
+          puts "\nFound #{values.length} unique values for field[#{index}] '#{header}'"
+        end
+      end
+    else
+      headers = fields
+      headers << 'lac-ci'
+      file_stats = fields.map{|h| {}}
+      file_stats << {}
+    end
+  end
+
+  if headers
+    found=[]
+    empty=[]
+    headers.each_with_index do |header,index|
+      stats=file_stats[index]
+      found << [header,stats] if(stats.keys.length>0)
+    end
+    found.each do |ff|
+      header,stats=*ff
+      output = STDOUT
+      filename = ([$output_prefix,header,$exclude_stats ? "Values" : "Stats"]).compact.join('_')+".txt"
+      if $output_files
+        output = File.open(filename,'w')
+      end
+      values = stats.keys
+      perc = 100.0 * values.length.to_f / lines.to_f
+      puts "Found #{values.length} unique values for field '#{header}'"
+      if !$show_all && (values.length > 500)
+        puts "\tNot printing more values more diverse than 500"
+      elsif (!$show_all && (perc > 75))
+        puts "\tNot printing more values more diverse than #{perc}%"
+      else
+        output.puts header
+        values.sort.each do |value|
+          value_text = (value.to_s.length < 1) ? '<empty>' : value
+          if $exclude_stats
+            output.puts "\t#{value_text}"
+          else
+            output.puts "\t#{value_text}\t#{stats[value]}"
+          end
+        end
+      end
+      if $output_files
+        puts "\tSaved to #{filename}"
+        output.close
+      else
+        puts
+      end
+    end
+  else
+    puts "No headers found in file #{file}"
+  end
+
+end
+
+

data/bin/show_geoptima

@@ -7,7 +7,7 @@ $: << '../lib'
 require 'date'
 require 'geoptima'
 
-Geoptima::assert_version("0.0.6")
+Geoptima::assert_version("0.0.7")
 
 $debug=false
 
@@ -15,10 +15,6 @@ $event_names=[]
 $files = []
 $print_limit = 10000
 
-def cw(val)
-  val.nil? ? '' : "(#{val})"
-end
-
 while arg=ARGV.shift do
   if arg =~ /^\-(\w+)/
     $1.split(//).each do |aa|

data/bin/show_geoptima_sos

@@ -0,0 +1,50 @@
+#!/usr/bin/env ruby
+
+# useful if being run inside a source code checkout
+$: << 'lib'
+$: << '../lib'
+
+require 'date'
+require 'geoptima'
+
+$debug=false
+
+$events={}
+$files = []
+
+$events={}
+
+while arg=ARGV.shift:
+  if arg =~ /^\-(\w+)/
+    $1.split(//).each do |aa|
+      case aa
+      when 'd'
+        $debug=true
+      else
+        puts "Unrecognized option: -#{aa}"
+      end
+    end
+  else
+    if File.exist? arg
+      $files << arg
+    else
+      puts "No such file: #{arg}"
+    end
+  end
+end
+
+$datasets = Geoptima::Dataset.make_datasets($files, :locate => true)
+
+puts "Found #{$datasets.length} IMEIs"
+$datasets.keys.sort.each do |imei|
+  dataset = $datasets[imei]
+  events = dataset.sorted('mode')
+  puts "\nFor #{imei} found #{dataset.length} events:\n"
+  if events && events.length>0
+    puts "Time\tLat\tLon\t#{events.first.header[1..-1].join("\t")}"
+    events.each do |event|
+      puts "#{event.time}\t#{event.latitude}\t#{event.longitude}\t#{event.data[1..-1].join("\t")}"
+    end
+  end
+end
+

data/examples/csv_chart.rb

@@ -0,0 +1,132 @@
+#!/usr/bin/ruby
+
+# useful if being run inside a source code checkout
+$: << 'lib'
+$: << '../lib'
+
+require 'geoptima/chart'
+require 'geoptima/options'
+
+$files = Geoptima::Options.process_args
+
+class Stats
+  attr_reader :file, :imei, :headers, :stats, :data
+  def initialize(file,imei,fields)
+    @file = file
+    @imei = imei
+    @headers = fields
+    @stats = fields.map{|h| {}}
+    @data = fields.map{|h| []}
+    @numerical = fields.map{|h| true}
+  end
+  def add(field,index)
+    if field =~ /\w/
+      @numerical[index] &&= is_number?(field)
+      puts "\tField[#{index}]: #{field}" if($debug)
+      stats = @stats[index]
+      stats[field] ||= 0
+      stats[field] += 1
+      puts "\tField[#{index}]: #{field} => #{stats[field]}" if($debug)
+      @data[index] << field
+    end
+  end
+  def is_number?(field)
+    is_integer?(field) || is_float?(field)
+  end
+  def is_integer?(field)
+    field.to_i.to_s == field
+  end
+  def is_float?(field)
+    field.to_f.to_s == field
+  end
+  def length(index)
+    @stats[index].length
+  end
+  def diversity(index)
+    100.0 * @stats[index].length.to_f / @data[index].length.to_f
+  end
+  def diverse?(index)
+    @stats[index].length>500 || diversity(index) > 40.0
+  end
+  def numerical?(index)
+    @numerical[index]
+  end
+end
+
+$stats = {}
+
+$files.each do |file|
+  lines = 0
+  imei = file.split(/[_\.]/)[0]
+  File.open(file).each do |line|
+    lines += 1
+    fields=line.chomp.split(/\t/)
+    if $stats[file]
+      puts "Processing line: #{line}" if($debug)
+      fields.each_with_index do |field,index|
+        $stats[file].add(field,index)
+      end
+    else
+      $stats[file] = Stats.new(file,imei,fields)
+    end
+  end
+end
+
+$stats.each do |file,stats|
+  stats.headers.each_with_index do |header,index|
+    puts "Charting #{header} with diversity #{stats.diversity(index)}"
+    case header
+    when 'signal.strength'
+      Geoptima::Chart.draw_line_chart(
+        stats.imei,
+        stats.data[0],
+        stats.data[index].map{|f| v=f.to_i; (v>-130 && v<0) ? v : nil},
+        :title => 'Signal Strength',
+        :maximum_value => -30,
+        :minimum_value => -130,
+        :width => 1024
+      ).write("Chart_#{stats.imei}_#{header}.png")
+
+      hist = stats.stats[index]
+      keys = hist.keys.sort{|a,b| a.to_i <=> b.to_i}
+      values = keys.map{|k| hist[k]}
+      Geoptima::Chart.draw_histogram_chart(
+        stats.imei, keys, values,
+        :title => 'Signal Strength Distribution',
+        :width => 1024
+      ).write("Chart_#{stats.imei}_#{header}_distribution.png")
+
+    when 'Event'
+      hist = stats.stats[index]
+      keys = hist.keys.sort{|a,b| a.to_i <=> b.to_i}
+      values = keys.map{|k| hist[k]}
+      Geoptima::Chart.draw_category_chart(
+        stats.imei, keys, values,
+        :title => "#{header} Distribution",
+        :width => 1024
+      ).write("Chart_#{stats.imei}_#{header}_distribution.png")
+
+    else
+      if stats.diverse?(index)
+        puts "Ingnoring high diversity field #{header}"
+      else
+        puts "Charting field: #{header} with length #{stats.length(index)} and diversity #{stats.diversity(index)}"
+        hist = stats.stats[index]
+        keys = hist.keys.sort{|a,b| a.to_i <=> b.to_i}
+        values = keys.map{|k| hist[k]}
+        args = [stats.imei, keys, values, {
+          :title => "#{header} Distribution",
+          :width => 1024}]
+        g = (stats.length(index) > 50) ?
+              Geoptima::Chart.draw_line_chart(*args) :
+            (stats.length(index) > 10 || stats.numerical?(index)) ?
+              Geoptima::Chart.draw_histogram_chart(*args) :
+            (stats.length(index) > 1) ?
+              Geoptima::Chart.draw_category_chart(*args) :
+            nil
+        g && g.write("Chart_#{stats.imei}_#{header}_distribution.png")
+      end
+    end
+  end
+end
+

data/examples/csv_stats.rb

@@ -0,0 +1,126 @@
+#!/usr/bin/env ruby
+
+# useful if being run inside a source code checkout
+$: << 'lib'
+$: << '../lib'
+
+require 'geoptima/options'
+
+$show_all = false
+
+$files = Geoptima::Options.process_args do |option|
+  option.a {$show_all = true}
+  option.o {$output_files = true}
+  option.u {$exclude_stats = true}
+  option.I {$include_props = ARGV.shift.split(/[\,\;]+/)}
+  option.O {$output_prefix = ARGV.shift}
+end
+
+$help = true if($files.length<0)
+if $help
+  puts <<EOHELP
+Usage: ./stats.rb <-dha> <-I include_props> files
+ -d  debug mode #{cw $debug}
+ -h  print this help #{cw $help}
+ -a  show all values, not just discrete ones #{cw $show_all}
+ -o  output the stats to files instead of the console #{cw $output_files}
+ -u  exclude stats, showing only unique lists of values #{cw $exclude_stats}
+ -I  only calculate statistics for named properties #{aw $include_props}
+ -O  prefix output files with specific prefix #{aw $output_prefix}
+EOHELP
+  exit 0
+end
+
+$files.each do |file|
+  lines = 0
+  headers = nil
+  file_stats = nil
+  File.open(file).each do |line|
+    lines += 1
+    fields=line.chomp.split(/[\t\,]/)
+    if headers
+      puts "Processing line: #{line}" if($debug)
+      lac,ci=nil
+      fields.each_with_index do |field,index|
+        if $include_props.to_s=='' || $include_props.index(headers[index])
+          puts "\tField[#{index}]: #{field}" if($debug)
+          stats = file_stats[index]
+          stats[field] ||= 0
+          stats[field] += 1
+          puts "\tField[#{index}]: #{field} => #{stats[field]}" if($debug)
+          lac=field if(headers[index]=='LAC' || headers[index]=='service.lac')
+          ci=field if(headers[index]=='CI' || headers[index]=='service.cell_id')
+          #puts "\tSet LAC=#{lac}, CI=#{ci} based on header #{headers[index]}" if($debug)
+        end
+      end
+      puts "\tSet LAC=#{lac}, CI=#{ci}" if($debug)
+      if lac && ci
+        index = headers.length - 1
+        puts "\tAdding statistics for LAC=#{lac}, CI=#{ci} using additional header '#{headers[index]}'" if($debug)
+        stats = file_stats[index]
+        field="#{lac}-#{ci}"
+        stats[field] ||= 0
+        stats[field] += 1
+        puts "\tField[#{index}]: #{field} => #{stats[field]}" if($debug)
+      end
+      if $debug
+        headers.each_with_index do |header,index|
+          stats = file_stats[index]
+          values = stats.keys
+          puts "\nFound #{values.length} unique values for field[#{index}] '#{header}'"
+        end
+      end
+    else
+      headers = fields
+      headers << 'lac-ci'
+      file_stats = fields.map{|h| {}}
+      file_stats << {}
+    end
+  end
+
+  if headers
+    found=[]
+    empty=[]
+    headers.each_with_index do |header,index|
+      stats=file_stats[index]
+      found << [header,stats] if(stats.keys.length>0)
+    end
+    found.each do |ff|
+      header,stats=*ff
+      output = STDOUT
+      filename = ([$output_prefix,header,$exclude_stats ? "Values" : "Stats"]).compact.join('_')+".txt"
+      if $output_files
+        output = File.open(filename,'w')
+      end
+      values = stats.keys
+      perc = 100.0 * values.length.to_f / lines.to_f
+      puts "Found #{values.length} unique values for field '#{header}'"
+      if !$show_all && (values.length > 500)
+        puts "\tNot printing more values more diverse than 500"
+      elsif (!$show_all && (perc > 75))
+        puts "\tNot printing more values more diverse than #{perc}%"
+      else
+        output.puts header
+        values.sort.each do |value|
+          value_text = (value.to_s.length < 1) ? '<empty>' : value
+          if $exclude_stats
+            output.puts "\t#{value_text}"
+          else
+            output.puts "\t#{value_text}\t#{stats[value]}"
+          end
+        end
+      end
+      if $output_files
+        puts "\tSaved to #{filename}"
+        output.close
+      else
+        puts
+      end
+    end
+  else
+    puts "No headers found in file #{file}"
+  end
+
+end
+
+

data/examples/show_geoptima.rb

@@ -7,7 +7,7 @@ $: << '../lib'
 require 'date'
 require 'geoptima'
 
-Geoptima::assert_version("0.0.6")
+Geoptima::assert_version("0.0.7")
 
 $debug=false
 
@@ -15,10 +15,6 @@ $event_names=[]
 $files = []
 $print_limit = 10000
 
-def cw(val)
-  val.nil? ? '' : "(#{val})"
-end
-
 while arg=ARGV.shift do
   if arg =~ /^\-(\w+)/
     $1.split(//).each do |aa|

data/geoptima.gemspec

@@ -25,7 +25,7 @@ EOF
   s.files        = Dir.glob("{bin,lib,rdoc}/**/*").reject{|x| x=~/(tmp|target|test-data)/ || x=~/~$/} +
                    Dir.glob("examples/*rb") + Dir.glob("examples/sample*json") + 
                    %w(README.rdoc CHANGELOG CONTRIBUTORS Gemfile geoptima.gemspec)
-  s.executables  = ['show_geoptima','geoptima_file_time']
+  s.executables  = ['show_geoptima','geoptima_file_time','csv_chart','csv_stats']
 
   s.extra_rdoc_files = %w( README.rdoc )
   s.rdoc_options = ["--quiet", "--title", "Geoptima.rb", "--line-numbers", "--main", "README.rdoc", "--inline-source"]

data/lib/geoptima/chart.rb

@@ -0,0 +1,119 @@
+#!/usr/bin/ruby
+
+require 'rubygems'
+
+$chart_libs = ['gruff'].map do |chart_lib|
+  begin
+    require chart_lib
+    chart_lib
+  rescue LoadError
+    puts "Failed to load charting library '#{chart_lib}': #{$!}"
+  end
+end.compact
+
+if $chart_libs.length > 0
+  puts "Loaded #{$chart_libs.length} charting libraries: #{$chart_libs.join(', ')}"
+else
+  puts "Warning: No charting libraries loaded. Chart support disabled."
+  puts "Please consider installing appropriate chart gems like 'gruff'"
+end
+
+module Geoptima
+  class Chart
+    DEFAULT_OPTIONS = {:show_points => true, :show_lines => true, :title => nil, :width => 800, :margins => 20, :font_size => 14}
+    attr_reader :chart_type
+    attr_accessor :chart, :data, :options
+    def initialize(chart_type,options={})
+      @chart_type = chart_type
+      @options = DEFAULT_OPTIONS.merge(options)
+    end
+    def self.engine
+      @@engine ||= $chart_libs[0] || 'not available'
+    end
+    def self.engine=(name)
+      @@engine = $chart_libs.grep(/name/)[0] || engine
+    end
+    def self.line(options={})
+      make_chart(:line,options)
+    end
+    def self.draw_line_chart(legend,keys,values,options={})
+      g = make_chart(:line,{:show_points => false}.merge(options))
+      g.data(legend, values)
+      g.labels= {0=>keys[0], (keys.length-1)=>keys[-1]}
+      options[:maximum_value] && g.maximum_value = options[:maximum_value].to_i
+      options[:minimum_value] && g.minimum_value = options[:minimum_value].to_i
+      options[:filename] && g.write(options[:filename])
+      g
+    end
+    def self.draw_histogram_chart(legend,keys,values,options={})
+      puts "Creating a chart with legend #{legend} for #{keys.length} keys and #{values.length} values"
+      g = make_chart(:bar, options)
+      g.data(legend, values)
+      g.minimum_value = 0
+      g.labels = keys.inject({}){|a,v| a[a.length] = v;a}
+      options[:filename] && g.write(options[:filename])
+      g
+    end
+    def self.draw_category_chart(legend,keys,values,options={})
+      puts "Creating category chart with keys: #{keys.join(',')}"
+      puts "Creating category chart with values: #{values.join(',')}"
+      g = make_chart(:bar, options)
+      keys.each_with_index do |key,index|
+        puts "\t Adding category #{key} with value #{values[index]}"
+        g.data(key, values[index])
+      end
+      g.minimum_value = 0
+      options[:filename] && g.write(options[:filename])
+      g
+    end
+    def self.bar(options={})
+      make_chart(:bar,options)
+    end
+    def self.make_chart(chart_type,options={})
+      case engine
+      when 'gruff'
+        GruffChart.new(chart_type,options)
+      else
+        puts "Unsupported chart engine: #{@@engine}"
+      end
+    end
+  end
+  class GruffChart < Chart
+    def initialize(chart_type,options={})
+      super(chart_type,options)
+    end
+    def chart
+      unless @chart
+        case chart_type.to_s
+        when 'line'
+          @chart = Gruff::Line.new(options[:width])
+          @chart.hide_dots = !options[:show_points]
+          @chart.hide_lines = !options[:show_lines]
+        when 'bar'
+          @chart = Gruff::Bar.new(options[:width])
+        else
+          raise "Unsupported chart type: #{chart_type}"
+        end
+        @chart.title = options[:title]
+        @chart.margins = options[:margins]
+        @chart.legend_font_size = options[:font_size]
+        @chart.marker_font_size = options[:font_size]
+        @chart.title_font_size = options[:title_font_size] || 2 * options[:font_size]
+      end
+      @chart
+    end
+    def data(name,values)
+      chart.data(name,values)
+    end
+    def labels=(label_map)
+      chart.labels = label_map
+    end
+    def method_missing(symbol,*args,&block)
+      chart.send symbol, *args, &block
+    end
+    def write(filename)
+      chart.write(filename)
+    end
+  end
+end
+

data/lib/geoptima/data.rb

@@ -3,6 +3,20 @@
 require 'rubygems'
 require 'multi_json'
 require 'date'
+if $use_dateperformance
+  begin
+    require 'date/performance'
+    require 'date/memoize'
+    class DateTime
+      def >(other) ; self - other > 0 ; end
+      def <(other) ; self - other < 0 ; end
+      def >=(other); self - other >= 0; end
+      def <=(other); self - other <= 0; end
+    end
+  rescue LoadError
+    puts "No date-performance gem installed, some features will run slower"
+  end
+end
 
 #
 # The Geoptima Module provides support for the Geoptima Client JSON file format
@@ -29,6 +43,26 @@ module Geoptima
     end
   end
 
+  class DateRange
+    attr_reader :min, :max, :range
+    def initialize(min,max)
+      @min = min
+      @max = max
+      @range = Range.new(min,max)
+    end
+    if ENV['RUBY_VERSION'] =~ /1\.8/
+      puts "Defining Range.include? to wrap for 1.8"
+      def include?(time)
+        @range.include?(time)
+      end
+    else
+      puts "Defining Range.include? to perform inequality tests for 1.9"
+      def include?(time)
+        (time >= min) && (time <= @max)
+      end
+    end
+  end
+
   # The Geoptima::Event class represents and individual record or event
   class Event
     KNOWN_HEADERS={
@@ -67,7 +101,7 @@ module Geoptima
       time.new_offset(0)
     end
     def time_key
-      utc.strftime("%Y-%m-%d %H:%M:%S.%3N")
+      utc.strftime("%Y-%m-%d %H:%M:%S.%3N").gsub(/\.(\d{3})\d+/,'.\1')
     end
     def [](key)
       @fields[key] || @fields[key.gsub(/#{name}\./,'')]
@@ -127,7 +161,7 @@ module Geoptima
       @fields[key] ||= subscriber[key] || subscriber[key.downcase]
     end
     def start
-      @start ||= subscriber['start'] && DateTime.parse(subscriber['start'].gsub(/Asia\/Bangkok/,'GMT+7').gsub(/Mar 17 2044/,'Feb 14 2012'))
+      @start ||= subscriber['start'] && DateTime.parse(subscriber['start'].gsub(/Asia\/Bangkok/,'GMT+7'))#.gsub(/Mar 17 2044/,'Feb 14 2012'))
     end
     def valid?
       start && start > Data.min_start && start < Data.max_start
@@ -216,7 +250,7 @@ module Geoptima
       @imei = imei
       @data = []
       @options = options
-      @time_range = options[:time_range] || Range.new(Config[:min_datetime],Config[:max_datetime])
+      @time_range = options[:time_range] || DateRange.new(Config[:min_datetime],Config[:max_datetime])
       @fields = {}
     end
 
@@ -347,17 +381,30 @@ module Geoptima
       @sorted ||= {}
       unless @sorted[nil]
         event_hash = {}
+        puts "Creating sorted maps for #{self}" if($debug)
         events_names.each do |name|
+          puts "Preparing maps for #{name}" if($debug)
           @data.each do |data|
+            puts "Processing #{(e=data.events[name]) && e.length} events for #{name}" if($debug)
             (events = data.events[name]) && events.each do |event|
+#              t = event.time.to_i
+              puts "\t\tTesting #{event.time} inside #{@time_range}" if($debug)
+#              if t < tmax
+#              if event.time > @time_range.min
+#              if (event.time >= @time_range.min) && (event.time < @time_range.max)
               if @time_range.include?(event.time)
+#              if @time_range.cover?(event.time)
+                puts "\t\t\tEvent at #{event.time} is inside #{@time_range}" if($debug)
                 key = "#{event.time_key} #{name}"
                 event_hash[key] = event
               end
             end
           end
+          puts "After adding #{name} events, maps are #{event_hash.length} long" if($debug)
         end
+        puts "Merging and sorting #{event_hash.keys.length} maps" if($debug)
         @sorted[nil] = event_hash.keys.sort.map{|k| event_hash[k]}
+        puts "Sorted #{@sorted[nil].length} events" if($debug)
         locate_events if(options[:locate])
       end
       @sorted
@@ -365,18 +412,21 @@ module Geoptima
 
     def locate_events
       prev_gps = nil
+      count = 0
+      puts "Locating #{sorted.length} events" if($debug)
       sorted.each do |event|
         if event.name === 'gps'
           event.locate(event)
           prev_gps = event
         elsif prev_gps
-          event.locate_if_closer_than(prev_gps,60)
+          count += 1 if(event.locate_if_closer_than(prev_gps,60))
         end
       end
+      puts "Located #{count} / #{sorted.length} events" if($debug)
     end
 
     def to_s
-      "IMEI:#{imei}, IMSI:#{imsis.join(',')}, Platform:#{platform}, Model:#{model}, OS:#{os}, Files:#{file_count}, Events:#{sorted.length}"
+      "IMEI:#{imei}, IMSI:#{imsis.join(',')}, Platform:#{platform}, Model:#{model}, OS:#{os}, Files:#{file_count}, Events:#{sorted && sorted.length}"
     end
 
     def self.make_datasets(files, options={})

data/lib/geoptima/options.rb

@@ -0,0 +1,67 @@
+#!/usr/bin/ruby
+
+def cw(val)
+  val.nil? ? '' : "(#{val})"
+end
+
+def aw(val)
+  val.nil? ? '' : "#{val.inspect}"
+end
+
+module Geoptima
+
+  class Options
+
+    attr_reader :args, :options, :debug
+
+    def initialize(debug=nil)
+      @debug = debug
+      @args = []
+      @options = {}
+    end
+    def add(*args,&block)
+      puts "Adding option processing for: #{args[0]}" if(debug)
+      @options[args[0].to_s] = block
+    end
+    def method_missing(symbol,*args,&block)
+      puts "Adding option processing for: #{symbol}" if(debug)
+      @options[symbol.to_s] = block
+    end
+    def process(a)
+      puts "Looking for match to option #{a}" if(debug)
+      @options.each do |opt,block|
+        puts "Comparing option #{a} to known option #{opt}" if(debug)
+        if opt === a
+          puts "Calling block for option #{a}: #{block.inspect}" if(debug)
+          block.call
+          return
+        end
+      end
+      puts "Unknown option: -#{a}"
+    end
+    def to_s
+      "Options[#{@options.keys.sort.join(', ')}]: #{args.join(', ')}"
+    end
+
+    def self.process_args(debug=nil)
+      options = Options.new(debug)
+      options.add('d') {$debug = true}
+      options.add('h') {$help = true}
+      puts "Processing options: #{options}" if(debug)
+      yield options if(block_given?)
+      while arg = ARGV.shift do
+        if arg =~ /^\-(\w+)/
+          $1.split(//).each do |a|
+            options.process a
+          end
+        else
+          options.args << arg
+        end
+      end
+      options.args
+    end
+
+  end
+
+end
+

data/lib/geoptima/version.rb

@@ -1,6 +1,6 @@
 module Geoptima
 
-  VERSION = "0.0.6"
+  VERSION = "0.0.7"
 
   def self.version_as_int(ver)
     base = 1

data/lib/geoptima.rb

@@ -1,3 +1,4 @@
 require 'geoptima/version.rb'
 require 'geoptima/data.rb'
+require 'geoptima/options.rb'
 

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: geoptima
 version: !ruby/object:Gem::Version 
-  hash: 19
+  hash: 17
   prerelease: 
   segments: 
   - 0
   - 0
-  - 6
-  version: 0.0.6
+  - 7
+  version: 0.0.7
 platform: ruby
 authors: 
 - Craig Taverner
@@ -15,7 +15,8 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-03-13 00:00:00 Z
+date: 2012-03-19 00:00:00 +01:00
+default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: multi_json
@@ -62,27 +63,35 @@ email: craig@amanzi.com
 executables: 
 - show_geoptima
 - geoptima_file_time
+- csv_chart
+- csv_stats
 extensions: []
 
 extra_rdoc_files: 
 - README.rdoc
 files: 
+- bin/show_geoptima_sos
 - bin/show_geoptima
+- bin/csv_chart
 - bin/geoptima_file_time
+- bin/csv_stats
 - lib/geoptima/version.rb
 - lib/geoptima/data.rb
+- lib/geoptima/chart.rb
+- lib/geoptima/options.rb
 - lib/geoptima.rb
 - examples/show_geoptima_sos.rb
-- examples/export_layer.rb
-- examples/stats.rb
 - examples/show_geoptima.rb
+- examples/csv_chart.rb
 - examples/geoptima_file_time.rb
+- examples/csv_stats.rb
 - examples/sample_geoptima.json
 - README.rdoc
 - CHANGELOG
 - CONTRIBUTORS
 - Gemfile
 - geoptima.gemspec
+has_rdoc: true
 homepage: http://github.com/craigtaverner/geoptima.rb
 licenses: []
 
@@ -120,7 +129,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
 requirements: []
 
 rubyforge_project: geoptima
-rubygems_version: 1.7.2
+rubygems_version: 1.6.2
 signing_key: 
 specification_version: 3
 summary: Ruby access to Geoptima JSON files

data/examples/export_layer.rb

@@ -1,59 +0,0 @@
-#!/usr/bin/env jruby
-
-# useful if being run inside a source code checkout
-$: << 'lib'
-$: << '../lib'
-
-require 'rubygems'
-require 'neo4j/spatial'
-require 'neo4j/spatial/cmd'
-
-$zoom = 1.0
-$args = Neo4j::Spatial::Cmd.args
-
-if $list === 'layers'
-  layers = Neo4j::Spatial::Layer.list
-  puts "Have #{layers.length} existing layers in the database:"
-  layers.each {|l| puts "\t#{l} (#{l.type_name})"}
-  puts
-  exit 0
-end
-
-if $help || $args.length < 1
-  puts <<-eos
-
-usage: ./export_layer.rb <-D storage_path> <-F format> <-E dir> <-Z zoom> <-W width> <-H height> <-l> <-h> layer <layers>
-    -D  Use specified database location
-    -F  Use specified export format (png, shp)
-    -E  Use specified export directory path (default '.')
-    -Z  Zoom in by specified factor (eg. 3.0)
-    -W  Image width (default 600)
-    -H  Image height (default 400)
-    -l  List existing database layers first
-    -h  Display this help and exit
-  The layer(s) should be pre-existing layers (including dynamic layers) in the database.
-  Supported formats are 'shp' for ESRI Shapefile and 'png' for images.
-
-For example:
-  ./export_layer.rb -D db -E exports -F png croatia.osm highway highway-residential natural-water
-
-This will export four previously defined layers to png format files in the 'exports' directory.
-
-eos
-  exit
-end
-
-if $format.to_s.downcase === 'shp'
-  $exporter = Neo4j::Spatial::SHPExporter.new :dir => $export
-else
-  $exporter = Neo4j::Spatial::ImageExporter.new :dir => $export, :zoom => $zoom, :width => $width, :height => $height
-end
-
-puts "Exporting #{$args.length} layers to #{$exporter.format}"
-
-$args.each do |layer|
-  l = Neo4j::Spatial::Layer.find layer
-  puts "Exporting #{l} (#{l.type_name}) - #{l.index.layer_bounding_box}"
-  $exporter.export l.name
-  puts "Finished exporting #{l} (#{l.type_name}) of #{l.index.count} entries"
-end

data/examples/stats.rb

@@ -1,41 +0,0 @@
-#!/usr/bin/env ruby
-
-$debug = false
-$lines = 0
-
-ARGF.each do |line|
-  $lines += 1
-  fields=line.chomp.split(/\t/)
-  if $headers
-    puts "Processing line: #{line}" if($debug)
-    fields.each_with_index do |field,index|
-      puts "\tField[#{index}]: #{field}" if($debug)
-      stats = $stats[index]
-      stats[field] ||= 0
-      stats[field] += 1
-      puts "\tField[#{index}]: #{field} => #{stats[field]}" if($debug)
-    end
-  else
-    $headers = fields
-    $stats = fields.map{|h| {}}
-  end
-end
-
-$headers.each_with_index do |header,index|
-  stats = $stats[index]
-  values = stats.keys
-  perc = 100.0 * values.length.to_f / $lines.to_f
-  puts "\nFound #{values.length} unique values for field '#{header}'"
-  if values.length > 500
-    puts "\tNot printing more values more diverse than 500"
-  elsif (perc > 75)
-    puts "\tNot printing more values more diverse than #{perc}%"
-  else
-    puts header
-    values.sort.each do |value|
-      value_text = (value.to_s.length < 1) ? '<empty>' : value
-      puts "\t#{value_text}\t#{stats[value]}"
-    end
-  end
-end
-