geoptima 0.0.9 → 0.1.0

data/bin/csv_chart

@@ -8,8 +8,10 @@ require 'geoptima/chart'
 require 'geoptima/version'
 require 'geoptima/options'
 require 'fileutils'
+require 'geoptima/daterange'
 
-Geoptima::assert_version("0.0.9")
+Geoptima::assert_version("0.1.0")
+Geoptima::Chart.available? || puts("No charting libraries available") || exit(-1)
 
 $export_dir = '.'
 $diversity = 40.0
@@ -17,14 +19,22 @@ $diversity = 40.0
 $files = Geoptima::Options.process_args do |option|
   option.m {$merge_all = true}
   option.a {$create_all = true}
+  option.t {$time_split = true}
   option.D {$export_dir = ARGV.shift}
   option.N {$merged_name = ARGV.shift}
   option.S {$specfile = ARGV.shift}
   option.P {$diversity = ARGV.shift.to_f}
+  option.T do
+    $time_range = Geoptima::DateRange.new(*(ARGV.shift.split(/[\,]+/).map do |t|
+      DateTime.parse t
+    end))
+  end
 end
 
 FileUtils.mkdir_p $export_dir
 
+$create_all = true unless($specfile)
+$merge_all = true if($time_split)
 $help = true unless($files.length>0)
 if $help
   puts <<EOHELP
@@ -33,44 +43,46 @@ Usage: csv_chart <-dham> <-S specfile> <-N name> <-D dir> files...
  -h  print this help #{cw $help}
  -a  automatically create charts for all properties #{cw $create_all}
  -m  merge all files into single stats #{cw $merge_all}
+ -t  merge and split by time (days) #{cw $time_split}
  -N  use specified name for merged dataset: #{$merged_name}
  -D  export charts to specified directory: #{$export_dir}
  -S  use chart specification in specified file: #{$specfile}
  -P  diversity threshold in percentage for automatic reports: #{$diversity}
+ -T  set time-range filter: #{$time_range}
 Files to import: #{$files.join(', ')}
 EOHELP
   exit
 end
 
 class Stats
-  attr_reader :file, :name, :headers, :stats, :data
-  def initialize(file,name,fields)
-    @file = file
+  attr_reader :name, :stats, :data, :numerical
+  def initialize(name)
     @name = name
-    @headers = fields
-    @stats = fields.map{|h| {}}
-    @data = fields.map{|h| []}
-    @numerical = fields.map{|h| true}
-  end
-  def add_header(h)
-    @headers << h
-    @stats << {}
-    @data << []
-    @numerical << true
-    @headers.length - 1
+    @stats = {}
+    @data = []
+    @numerical = true
   end
-  def add(field,index)
-    puts "\tAdding field '#{field}' at index #{index}" if($debug)
+  def add(field)
+    puts "\tAdding field '#{field}' for property #{name}" if($debug)
     if field && field =~ /\w/
-      @numerical[index] &&= is_number?(field)
-      puts "\tField[#{index}]: #{field}" if($debug)
-      stats = @stats[index]
+      @numerical &&= is_number?(field)
       stats[field] ||= 0
       stats[field] += 1
-      puts "\tField[#{index}]: #{field} => #{stats[field]}" if($debug)
-      @data[index] << field
+      @data << field
     end
   end
+  def length
+    @stats.length
+  end
+  def numerical?
+    @numerical
+  end
+  def diversity
+    100.0 * @stats.length.to_f / @data.length.to_f
+  end
+  def diverse?
+    @stats.length>500 || diversity > $diversity
+  end
   def is_number?(field)
     is_integer?(field) || is_float?(field)
   end
@@ -80,36 +92,104 @@ class Stats
   def is_float?(field)
     field.to_f.to_s == field
   end
-  def length(index)
-    @stats[index].length
+  def to_s
+    "#{name}[#{length}]"
+  end
+end
+
+class StatsManager
+  attr_reader :name, :headers, :stats
+  def initialize(name)
+    @name = name
+    @headers = []
+    @stats = {}
+  end
+  def time_index
+    @time_index ||= @headers.index('Time') || @headers.index('Timestamp')
+  end
+  def time_stats
+    @time_stats ||= get_stats('Time') || get_stats('Timestamp')
+  end
+  def set_headers(headers)
+    @headers = []
+    headers.each {|h| add_header(h)}
+    $specs && $specs.add_stats(self,headers)
+  end
+  def add_header(h)
+    if @headers.index(h)
+      puts "Stats header already exists: #{h}"
+    else
+      @headers << h
+      @stats[h] ||= Stats.new(h)
+    end
+    @headers.index(h)
+  end
+  def get_stats(header)
+    stats[header] || stats[header.downcase]
+  end
+  def add_all(fields,headers)
+    fields.each_with_index do |field,index|
+      add(field,headers[index])
+    end
+    $specs && $specs.add_fields(self,fields)
   end
-  def diversity(index)
-    100.0 * @stats[index].length.to_f / @data[index].length.to_f
+  def add(field,header)
+    puts "\tAdding field '#{field}' for property #{header}" if($debug)
+    add_header(header) unless(@stats[header])
+    @stats[header].add(field)
   end
-  def diverse?(index)
-    @stats[index].length>500 || diversity(index) > $diversity
+  def length
+    @stats.length
   end
-  def numerical?(index)
-    @numerical[index]
+  def to_s
+    "Stats[#{length}]: #{@stats.inspect}"
   end
 end
 
-$stats = {}
-
 module Geoptima
   class StatSpec
-    attr_reader :header, :index, :indices, :fields, :options, :proc
+    attr_reader :header, :index, :indices, :fields, :options, :proc, :groups
     def initialize(header,*fields,&block)
       @header = header
       @fields = fields
       @proc = block
+      @groups = {}
       if @fields[-1].is_a?(Hash)
         @options = @fields.pop
       else
         @options = {}
       end
+      if @options[:group]
+        case @options[:group].to_s.intern
+        when :months
+          group_by {|t| t.strftime("%Y-%m")}
+        when :days
+          group_by {|t| t.strftime("%Y-%m-%d")}
+        else
+          group_by {|t| t.strftime("%Y-%m-%d %H")}
+        end
+      end
       puts "Created StatSpec: #{self}"
     end
+    def group_by(&block)
+      @group = block
+    end
+    def add(stats_manager,fields)
+      if @group
+        begin
+          time = DateTime.parse(fields[stats_manager.time_index])
+          if $time_range.nil? || $time_range.include?(time)
+            key = @group.call(time)
+            ghead = "#{header} #{key}"
+            @groups[key] = ghead
+            stats_manager.add(map(fields),ghead)
+          end
+        rescue ArgumentError
+          puts "Error: Unable to process time field[#{time}]: #{$!}"
+        end
+      end
+      stats_manager.add(map(fields),header)
+    end
     def mk_range(val)
       if val =~ /\w/
         div = options[:div].to_i
@@ -120,7 +200,7 @@ module Geoptima
         val
       end
     end
-    def map(values)
+    def map(values,filter=nil)
       if @indices
         puts "StatSpec[#{self}]: #{options.inspect}" if($debug)
         vals = @indices.map{|i| values[i]}
@@ -133,12 +213,14 @@ module Geoptima
         val
       end
     end
-    def prepare_indices(stats)
-      if stats.headers.index(header)
+    def prepare_indices(stats_manager,headers)
+      if headers.index(header)
         puts "Header '#{header}' already exists, cannot create #{self}"
+        @index = nil
+        @indices = nil
       else
-        @index = stats.add_header(header)
-        @indices = @fields.map{|h| stats.headers.index(h) || stats.headers.index(h.downcase) }
+        @index = stats_manager.add_header(header)
+        @indices = @fields.map{|h| headers.index(h) || headers.index(h.downcase) }
         puts "#{self}: Header[#{@index}], Indices[#{@indices.join(',')}]" if($debug)
         if @indices.compact.length < @fields.length
           puts "Unable to find some headers for #{self}, ignoring stats"
@@ -152,35 +234,60 @@ module Geoptima
   end
   class ChartSpec
     attr_reader :chart_type, :header, :options
-    def initialize(chart_type,header,options={})
+    def initialize(header,options={})
       @header = header
-      @chart_type = chart_type
+      @chart_type = options[:chart_type] || :histogram
       @options = options
     end
-    def process(stats)
-      puts "Charting #{header} using stats.headers: #{stats.headers}"
-      index = stats.headers.index(header)
-      index ||= stats.headers.index(header.downcase)
-      unless index
+    def process(stats_manager)
+      puts "Charting #{header} using headers: #{stats_manager.headers.inspect}"
+      stat_spec = $specs.stat_specs.find{|o| o.header == header}
+      stats = stats_manager.get_stats(header)
+      grouped_stats = {}
+      if stat_spec
+        stat_spec.groups.each do |name,header|
+          gs = stats_manager.get_stats(header)
+          grouped_stats[name] = gs
+          stats ||= gs
+        end
+      end
+      unless stats
         puts "Cannot find statistics for '#{header}' - ignoring chart"
         return
       end
-      puts "Charting #{header} at index #{index} and options #{options.inspect}"
-      puts "Charting #{header} with diversity #{stats.diversity(index)}"
-      hist = stats.stats[index]
-      title = options[:title]
-      if options[:top]
-        keys = hist.keys.sort{|a,b| hist[b] <=> hist[a]}[0..(options[:top].to_i)]
-        title ||= "#{header} Top #{options[:top]}"
+      puts "Charting #{header} with options #{options.inspect} and stats: #{stats}"
+      puts "Charting #{header} with diversity #{stats.diversity}"
+      if grouped_stats.length > 0
+        title = options[:title]
+        title ||= "#{header} Distribution"
+        options.merge!( :title => title, :width => 1024 )
+        value_map = {}
+        groups = grouped_stats.keys.sort
+        groups.each_with_index do |name,index|
+          gs = grouped_stats[name]
+          hist = gs.stats
+          hist.keys.each do |k|
+            value_map[k] ||= [].fill(0,0...groups.length)
+            value_map[k][index] = hist[k]
+          end
+        end
+        legends = value_map.keys.sort
+        g = Geoptima::Chart.draw_grouped_chart legends, groups, value_map, options
       else
-        keys = hist.keys.sort{|a,b| a.to_i <=> b.to_i}
+        hist = stats.stats
+        title = options[:title]
+        if options[:top]
+          keys = hist.keys.sort{|a,b| hist[b] <=> hist[a]}[0..(options[:top].to_i)]
+          title ||= "#{header} Top #{options[:top]}"
+        else
+          keys = hist.keys.sort{|a,b| a.to_i <=> b.to_i}
+        end
+        values = keys.map{|k| hist[k]}
+        title ||= "#{header} Distribution"
+        options.merge!( :title => title, :width => 1024 )
+        g = Geoptima::Chart.send "draw_#{chart_type}_chart", stats_manager.name, keys, values, options
       end
-      values = keys.map{|k| hist[k]}
-      title ||= "#{header} Distribution"
-      options.merge!( :title => title, :width => 1024 )
-      legend = $merge_all ? "ALL" : stats.name
-      g = Geoptima::Chart.send "draw_#{chart_type}_chart", stats.name, keys, values, options
-      g.write("#{$export_dir}/Chart_#{stats.name}_#{header}_distribution.png")
+      g.write("#{$export_dir}/Chart_#{stats_manager.name}_#{header}_#{chart_type}_distribution.png")
     end
     def to_s
       "#{chart_type.upcase}-#{header}"
@@ -194,126 +301,138 @@ module Geoptima
       instance_eval(File.open(specfile).read)
     end
     def category_chart(header,options={})
-      chart(:category, header, options)
+      chart(header, options.merge(:chart_type => :category))
     end
     def histogram_chart(header,options={})
-      chart(:histogram, header, options)
+      chart(header, options.merge(:chart_type => :histgram))
     end
     def line_chart(header,options={})
-      chart(:line, header, options)
+      chart(header, options.merge(:chart_type => :line))
     end
-    def chart(chart_type,header,options={})
-      @chart_specs << ChartSpec.new(chart_type,header,options)
+    def chart(header,options={})
+      @chart_specs << ChartSpec.new(header,options)
     end
     def stats(header,*fields,&block)
       @stat_specs << StatSpec.new(header,*fields,&block)
     end
-    def add_stats(stats)
+    def add_stats(stats_manager,headers)
       stat_specs.each do |stat_spec|
-        stat_spec.prepare_indices(stats)
+        stat_spec.prepare_indices(stats_manager,headers)
       end
     end
-    def add_fields(stats,fields)
-      $specs.stat_specs.each do |stat_spec|
-        stats.add(
-          stat_spec.map(fields),
-          stat_spec.index
-        )
+    def add_fields(stats_manager,fields)
+      stat_specs.each do |stat_spec|
+        stat_spec.add(stats_manager,fields)
       end
     end
     def to_s
-      "Charts[#{chart_specs.length}]: #{@chart_specs.join(', ')}"
-    end
-  end
-end
-
-$specfile && $specs = Geoptima::StatsSpecs.new($specfile)
-
-$files.each do |file|
-  lines = 0
-  filename = File.basename(file)
-  (names = filename.split(/[_\.]/)).pop
-  name = $merged_name || names.join('_')
-  puts "About to read file #{file}"
-  File.open(file).each do |line|
-    lines += 1
-    fields=line.chomp.split(/\t/)
-    if $stats[file]
-      puts "Processing line: #{line}" if($debug)
-      fields.each_with_index do |field,index|
-        $stats[file].add(field,index)
-      end
-      $specs && $specs.add_fields($stats[file],fields)
-    elsif($merge_all && $stats.length>0)
-      file = $stats.values[0].file
-    else
-      $stats[file] = Stats.new(filename,name,fields)
-      $specs && $specs.add_stats($stats[file])
+      "Stats[#{@stat_specs.join(', ')}] AND Charts[#{@chart_specs.join(', ')}]"
     end
   end
 end
 
-$stats.each do |file,stats|
-  if $specs
-    $specs.chart_specs.each do |chart_spec|
-      chart_spec.process(stats)
-    end
-  else
-  stats.headers.each_with_index do |header,index|
-    puts "Charting #{header} with diversity #{stats.diversity(index)}"
+def create_all(name,stats_manager)
+  stats_manager.headers.each do |header|
+    stats = stats_manager.stats[header]
+    puts "Charting #{header} with diversity #{stats.diversity}"
     case header
     when 'signal.strength'
       Geoptima::Chart.draw_line_chart(
-        stats.name,
-        stats.data[0],
-        stats.data[index].map{|f| v=f.to_i; (v>-130 && v<0) ? v : nil},
+        stats_manager.name,
+        stats_manager.time_stats.data,
+        stats.data.map{|f| v=f.to_i; (v>-130 && v<0) ? v : nil},
         :title => 'Signal Strength',
         :maximum_value => -30,
         :minimum_value => -130,
         :width => 1024
-      ).write("#{$export_dir}/Chart_#{stats.name}_#{header}.png")
+      ).write("#{$export_dir}/Chart_#{stats_manager.name}_#{header}.png")
 
-      hist = stats.stats[index]
+      hist = stats.stats
       keys = hist.keys.sort{|a,b| a.to_i <=> b.to_i}
       values = keys.map{|k| hist[k]}
       Geoptima::Chart.draw_histogram_chart(
-        stats.name, keys, values,
+        stats_manager.name, keys, values,
         :title => 'Signal Strength Distribution',
         :width => 1024
-      ).write("#{$export_dir}/Chart_#{stats.name}_#{header}_distribution.png")
+      ).write("#{$export_dir}/Chart_#{stats_manager.name}_#{header}_distribution.png")
 
     when 'Event'
-      hist = stats.stats[index]
+      hist = stats.stats
       keys = hist.keys.sort{|a,b| a.to_i <=> b.to_i}
       values = keys.map{|k| hist[k]}
       Geoptima::Chart.draw_category_chart(
-        stats.name, keys, values,
+        stats_manager.name, keys, values,
         :title => "#{header} Distribution",
         :width => 1024
-      ).write("#{$export_dir}/Chart_#{stats.name}_#{header}_distribution.png")
+      ).write("#{$export_dir}/Chart_#{stats_manager.name}_#{header}_distribution.png")
 
     else
-      if stats.diverse?(index)
+      if stats.diverse?
         puts "Ignoring high diversity field #{header}"
       else
-        puts "Charting field: #{header} with length #{stats.length(index)} and diversity #{stats.diversity(index)}"
-        hist = stats.stats[index]
+        puts "Charting field: #{header} with length #{stats.length} and diversity #{stats.diversity}"
+        hist = stats.stats
         keys = hist.keys.sort{|a,b| a.to_i <=> b.to_i}
         values = keys.map{|k| hist[k]}
-        args = [stats.name, keys, values, {
+        args = [stats_manager.name, keys, values, {
           :title => "#{header} Distribution",
           :width => 1024}]
-        g = (stats.length(index) > 50) ?
+        g = (stats.length > 50) ?
               Geoptima::Chart.draw_line_chart(*args) :
-            (stats.length(index) > 10 || stats.numerical?(index)) ?
+            (stats.length > 10 || stats.numerical?) ?
               Geoptima::Chart.draw_histogram_chart(*args) :
-            (stats.length(index) > 1) ?
+            (stats.length > 1) ?
               Geoptima::Chart.draw_category_chart(*args) :
             nil
-        g && g.write("#{$export_dir}/Chart_#{stats.name}_#{header}_distribution.png")
+        g && g.write("#{$export_dir}/Chart_#{stats_manager.name}_#{header}_distribution.png")
       end
     end
   end
+end
+
+#
+# Now run the actual program, reading the specifation file and then the CSV files
+#
+
+$stats_managers = {}
+
+$specfile && $specs = Geoptima::StatsSpecs.new($specfile)
+
+$files.each do |file|
+  lines = 0
+  headers = nil
+  filename = File.basename(file)
+  (names = filename.split(/[_\.]/)).pop
+  name = $merge_all ? ($merged_name || 'All') : names.join('_')
+  $stats_managers[name] ||= StatsManager.new(name)
+  puts "About to read file #{file}"
+  File.open(file).each do |line|
+    lines += 1
+    fields=line.chomp.split(/\t/)
+    if headers
+      puts "Processing line: #{line}" if($debug)
+      $stats_managers[name].add_all(fields,headers)
+    else
+      headers = fields
+      if headers.length<2
+        puts "Too few headers, rejecting #{file}"
+        break
+      end
+      $stats_managers[name].set_headers(headers)
+    end
+  end
+end
+
+# Finally output all charts specified
+
+$stats_managers.each do |name,stats_manager|
+  if $specs
+    $specs.chart_specs.each do |chart_spec|
+      chart_spec.process(stats_manager)
+    end
+  end
+  if $create_all
+    create_all name, stats_manager
   end
 end
 

data/bin/show_geoptima

@@ -7,7 +7,7 @@ $: << '../lib'
 require 'date'
 require 'geoptima'
 
-Geoptima::assert_version("0.0.9")
+Geoptima::assert_version("0.1.0")
 
 $debug=false
 

data/examples/csv_chart.rb

@@ -8,8 +8,10 @@ require 'geoptima/chart'
 require 'geoptima/version'
 require 'geoptima/options'
 require 'fileutils'
+require 'geoptima/daterange'
 
-Geoptima::assert_version("0.0.9")
+Geoptima::assert_version("0.1.0")
+Geoptima::Chart.available? || puts("No charting libraries available") || exit(-1)
 
 $export_dir = '.'
 $diversity = 40.0
@@ -17,14 +19,22 @@ $diversity = 40.0
 $files = Geoptima::Options.process_args do |option|
   option.m {$merge_all = true}
   option.a {$create_all = true}
+  option.t {$time_split = true}
   option.D {$export_dir = ARGV.shift}
   option.N {$merged_name = ARGV.shift}
   option.S {$specfile = ARGV.shift}
   option.P {$diversity = ARGV.shift.to_f}
+  option.T do
+    $time_range = Geoptima::DateRange.new(*(ARGV.shift.split(/[\,]+/).map do |t|
+      DateTime.parse t
+    end))
+  end
 end
 
 FileUtils.mkdir_p $export_dir
 
+$create_all = true unless($specfile)
+$merge_all = true if($time_split)
 $help = true unless($files.length>0)
 if $help
   puts <<EOHELP
@@ -33,44 +43,46 @@ Usage: csv_chart <-dham> <-S specfile> <-N name> <-D dir> files...
  -h  print this help #{cw $help}
  -a  automatically create charts for all properties #{cw $create_all}
  -m  merge all files into single stats #{cw $merge_all}
+ -t  merge and split by time (days) #{cw $time_split}
  -N  use specified name for merged dataset: #{$merged_name}
  -D  export charts to specified directory: #{$export_dir}
  -S  use chart specification in specified file: #{$specfile}
  -P  diversity threshold in percentage for automatic reports: #{$diversity}
+ -T  set time-range filter: #{$time_range}
 Files to import: #{$files.join(', ')}
 EOHELP
   exit
 end
 
 class Stats
-  attr_reader :file, :name, :headers, :stats, :data
-  def initialize(file,name,fields)
-    @file = file
+  attr_reader :name, :stats, :data, :numerical
+  def initialize(name)
     @name = name
-    @headers = fields
-    @stats = fields.map{|h| {}}
-    @data = fields.map{|h| []}
-    @numerical = fields.map{|h| true}
-  end
-  def add_header(h)
-    @headers << h
-    @stats << {}
-    @data << []
-    @numerical << true
-    @headers.length - 1
+    @stats = {}
+    @data = []
+    @numerical = true
   end
-  def add(field,index)
-    puts "\tAdding field '#{field}' at index #{index}" if($debug)
+  def add(field)
+    puts "\tAdding field '#{field}' for property #{name}" if($debug)
     if field && field =~ /\w/
-      @numerical[index] &&= is_number?(field)
-      puts "\tField[#{index}]: #{field}" if($debug)
-      stats = @stats[index]
+      @numerical &&= is_number?(field)
       stats[field] ||= 0
       stats[field] += 1
-      puts "\tField[#{index}]: #{field} => #{stats[field]}" if($debug)
-      @data[index] << field
+      @data << field
     end
   end
+  def length
+    @stats.length
+  end
+  def numerical?
+    @numerical
+  end
+  def diversity
+    100.0 * @stats.length.to_f / @data.length.to_f
+  end
+  def diverse?
+    @stats.length>500 || diversity > $diversity
+  end
   def is_number?(field)
     is_integer?(field) || is_float?(field)
   end
@@ -80,36 +92,104 @@ class Stats
   def is_float?(field)
     field.to_f.to_s == field
   end
-  def length(index)
-    @stats[index].length
+  def to_s
+    "#{name}[#{length}]"
+  end
+end
+
+class StatsManager
+  attr_reader :name, :headers, :stats
+  def initialize(name)
+    @name = name
+    @headers = []
+    @stats = {}
+  end
+  def time_index
+    @time_index ||= @headers.index('Time') || @headers.index('Timestamp')
+  end
+  def time_stats
+    @time_stats ||= get_stats('Time') || get_stats('Timestamp')
+  end
+  def set_headers(headers)
+    @headers = []
+    headers.each {|h| add_header(h)}
+    $specs && $specs.add_stats(self,headers)
+  end
+  def add_header(h)
+    if @headers.index(h)
+      puts "Stats header already exists: #{h}"
+    else
+      @headers << h
+      @stats[h] ||= Stats.new(h)
+    end
+    @headers.index(h)
+  end
+  def get_stats(header)
+    stats[header] || stats[header.downcase]
+  end
+  def add_all(fields,headers)
+    fields.each_with_index do |field,index|
+      add(field,headers[index])
+    end
+    $specs && $specs.add_fields(self,fields)
   end
-  def diversity(index)
-    100.0 * @stats[index].length.to_f / @data[index].length.to_f
+  def add(field,header)
+    puts "\tAdding field '#{field}' for property #{header}" if($debug)
+    add_header(header) unless(@stats[header])
+    @stats[header].add(field)
   end
-  def diverse?(index)
-    @stats[index].length>500 || diversity(index) > $diversity
+  def length
+    @stats.length
   end
-  def numerical?(index)
-    @numerical[index]
+  def to_s
+    "Stats[#{length}]: #{@stats.inspect}"
   end
 end
 
-$stats = {}
-
 module Geoptima
   class StatSpec
-    attr_reader :header, :index, :indices, :fields, :options, :proc
+    attr_reader :header, :index, :indices, :fields, :options, :proc, :groups
     def initialize(header,*fields,&block)
       @header = header
       @fields = fields
       @proc = block
+      @groups = {}
       if @fields[-1].is_a?(Hash)
         @options = @fields.pop
       else
         @options = {}
       end
+      if @options[:group]
+        case @options[:group].to_s.intern
+        when :months
+          group_by {|t| t.strftime("%Y-%m")}
+        when :days
+          group_by {|t| t.strftime("%Y-%m-%d")}
+        else
+          group_by {|t| t.strftime("%Y-%m-%d %H")}
+        end
+      end
       puts "Created StatSpec: #{self}"
     end
+    def group_by(&block)
+      @group = block
+    end
+    def add(stats_manager,fields)
+      if @group
+        begin
+          time = DateTime.parse(fields[stats_manager.time_index])
+          if $time_range.nil? || $time_range.include?(time)
+            key = @group.call(time)
+            ghead = "#{header} #{key}"
+            @groups[key] = ghead
+            stats_manager.add(map(fields),ghead)
+          end
+        rescue ArgumentError
+          puts "Error: Unable to process time field[#{time}]: #{$!}"
+        end
+      end
+      stats_manager.add(map(fields),header)
+    end
     def mk_range(val)
       if val =~ /\w/
         div = options[:div].to_i
@@ -120,7 +200,7 @@ module Geoptima
         val
       end
     end
-    def map(values)
+    def map(values,filter=nil)
       if @indices
         puts "StatSpec[#{self}]: #{options.inspect}" if($debug)
         vals = @indices.map{|i| values[i]}
@@ -133,12 +213,14 @@ module Geoptima
         val
       end
     end
-    def prepare_indices(stats)
-      if stats.headers.index(header)
+    def prepare_indices(stats_manager,headers)
+      if headers.index(header)
         puts "Header '#{header}' already exists, cannot create #{self}"
+        @index = nil
+        @indices = nil
       else
-        @index = stats.add_header(header)
-        @indices = @fields.map{|h| stats.headers.index(h) || stats.headers.index(h.downcase) }
+        @index = stats_manager.add_header(header)
+        @indices = @fields.map{|h| headers.index(h) || headers.index(h.downcase) }
         puts "#{self}: Header[#{@index}], Indices[#{@indices.join(',')}]" if($debug)
         if @indices.compact.length < @fields.length
           puts "Unable to find some headers for #{self}, ignoring stats"
@@ -152,35 +234,60 @@ module Geoptima
   end
   class ChartSpec
     attr_reader :chart_type, :header, :options
-    def initialize(chart_type,header,options={})
+    def initialize(header,options={})
       @header = header
-      @chart_type = chart_type
+      @chart_type = options[:chart_type] || :histogram
       @options = options
     end
-    def process(stats)
-      puts "Charting #{header} using stats.headers: #{stats.headers}"
-      index = stats.headers.index(header)
-      index ||= stats.headers.index(header.downcase)
-      unless index
+    def process(stats_manager)
+      puts "Charting #{header} using headers: #{stats_manager.headers.inspect}"
+      stat_spec = $specs.stat_specs.find{|o| o.header == header}
+      stats = stats_manager.get_stats(header)
+      grouped_stats = {}
+      if stat_spec
+        stat_spec.groups.each do |name,header|
+          gs = stats_manager.get_stats(header)
+          grouped_stats[name] = gs
+          stats ||= gs
+        end
+      end
+      unless stats
         puts "Cannot find statistics for '#{header}' - ignoring chart"
         return
       end
-      puts "Charting #{header} at index #{index} and options #{options.inspect}"
-      puts "Charting #{header} with diversity #{stats.diversity(index)}"
-      hist = stats.stats[index]
-      title = options[:title]
-      if options[:top]
-        keys = hist.keys.sort{|a,b| hist[b] <=> hist[a]}[0..(options[:top].to_i)]
-        title ||= "#{header} Top #{options[:top]}"
+      puts "Charting #{header} with options #{options.inspect} and stats: #{stats}"
+      puts "Charting #{header} with diversity #{stats.diversity}"
+      if grouped_stats.length > 0
+        title = options[:title]
+        title ||= "#{header} Distribution"
+        options.merge!( :title => title, :width => 1024 )
+        value_map = {}
+        groups = grouped_stats.keys.sort
+        groups.each_with_index do |name,index|
+          gs = grouped_stats[name]
+          hist = gs.stats
+          hist.keys.each do |k|
+            value_map[k] ||= [].fill(0,0...groups.length)
+            value_map[k][index] = hist[k]
+          end
+        end
+        legends = value_map.keys.sort
+        g = Geoptima::Chart.draw_grouped_chart legends, groups, value_map, options
       else
-        keys = hist.keys.sort{|a,b| a.to_i <=> b.to_i}
+        hist = stats.stats
+        title = options[:title]
+        if options[:top]
+          keys = hist.keys.sort{|a,b| hist[b] <=> hist[a]}[0..(options[:top].to_i)]
+          title ||= "#{header} Top #{options[:top]}"
+        else
+          keys = hist.keys.sort{|a,b| a.to_i <=> b.to_i}
+        end
+        values = keys.map{|k| hist[k]}
+        title ||= "#{header} Distribution"
+        options.merge!( :title => title, :width => 1024 )
+        g = Geoptima::Chart.send "draw_#{chart_type}_chart", stats_manager.name, keys, values, options
       end
-      values = keys.map{|k| hist[k]}
-      title ||= "#{header} Distribution"
-      options.merge!( :title => title, :width => 1024 )
-      legend = $merge_all ? "ALL" : stats.name
-      g = Geoptima::Chart.send "draw_#{chart_type}_chart", stats.name, keys, values, options
-      g.write("#{$export_dir}/Chart_#{stats.name}_#{header}_distribution.png")
+      g.write("#{$export_dir}/Chart_#{stats_manager.name}_#{header}_#{chart_type}_distribution.png")
     end
     def to_s
       "#{chart_type.upcase}-#{header}"
@@ -194,126 +301,138 @@ module Geoptima
       instance_eval(File.open(specfile).read)
     end
     def category_chart(header,options={})
-      chart(:category, header, options)
+      chart(header, options.merge(:chart_type => :category))
     end
     def histogram_chart(header,options={})
-      chart(:histogram, header, options)
+      chart(header, options.merge(:chart_type => :histgram))
     end
     def line_chart(header,options={})
-      chart(:line, header, options)
+      chart(header, options.merge(:chart_type => :line))
     end
-    def chart(chart_type,header,options={})
-      @chart_specs << ChartSpec.new(chart_type,header,options)
+    def chart(header,options={})
+      @chart_specs << ChartSpec.new(header,options)
     end
     def stats(header,*fields,&block)
       @stat_specs << StatSpec.new(header,*fields,&block)
     end
-    def add_stats(stats)
+    def add_stats(stats_manager,headers)
       stat_specs.each do |stat_spec|
-        stat_spec.prepare_indices(stats)
+        stat_spec.prepare_indices(stats_manager,headers)
       end
     end
-    def add_fields(stats,fields)
-      $specs.stat_specs.each do |stat_spec|
-        stats.add(
-          stat_spec.map(fields),
-          stat_spec.index
-        )
+    def add_fields(stats_manager,fields)
+      stat_specs.each do |stat_spec|
+        stat_spec.add(stats_manager,fields)
       end
     end
     def to_s
-      "Charts[#{chart_specs.length}]: #{@chart_specs.join(', ')}"
-    end
-  end
-end
-
-$specfile && $specs = Geoptima::StatsSpecs.new($specfile)
-
-$files.each do |file|
-  lines = 0
-  filename = File.basename(file)
-  (names = filename.split(/[_\.]/)).pop
-  name = $merged_name || names.join('_')
-  puts "About to read file #{file}"
-  File.open(file).each do |line|
-    lines += 1
-    fields=line.chomp.split(/\t/)
-    if $stats[file]
-      puts "Processing line: #{line}" if($debug)
-      fields.each_with_index do |field,index|
-        $stats[file].add(field,index)
-      end
-      $specs && $specs.add_fields($stats[file],fields)
-    elsif($merge_all && $stats.length>0)
-      file = $stats.values[0].file
-    else
-      $stats[file] = Stats.new(filename,name,fields)
-      $specs && $specs.add_stats($stats[file])
+      "Stats[#{@stat_specs.join(', ')}] AND Charts[#{@chart_specs.join(', ')}]"
     end
   end
 end
 
-$stats.each do |file,stats|
-  if $specs
-    $specs.chart_specs.each do |chart_spec|
-      chart_spec.process(stats)
-    end
-  else
-  stats.headers.each_with_index do |header,index|
-    puts "Charting #{header} with diversity #{stats.diversity(index)}"
+def create_all(name,stats_manager)
+  stats_manager.headers.each do |header|
+    stats = stats_manager.stats[header]
+    puts "Charting #{header} with diversity #{stats.diversity}"
     case header
     when 'signal.strength'
       Geoptima::Chart.draw_line_chart(
-        stats.name,
-        stats.data[0],
-        stats.data[index].map{|f| v=f.to_i; (v>-130 && v<0) ? v : nil},
+        stats_manager.name,
+        stats_manager.time_stats.data,
+        stats.data.map{|f| v=f.to_i; (v>-130 && v<0) ? v : nil},
         :title => 'Signal Strength',
         :maximum_value => -30,
         :minimum_value => -130,
         :width => 1024
-      ).write("#{$export_dir}/Chart_#{stats.name}_#{header}.png")
+      ).write("#{$export_dir}/Chart_#{stats_manager.name}_#{header}.png")
 
-      hist = stats.stats[index]
+      hist = stats.stats
       keys = hist.keys.sort{|a,b| a.to_i <=> b.to_i}
       values = keys.map{|k| hist[k]}
       Geoptima::Chart.draw_histogram_chart(
-        stats.name, keys, values,
+        stats_manager.name, keys, values,
         :title => 'Signal Strength Distribution',
         :width => 1024
-      ).write("#{$export_dir}/Chart_#{stats.name}_#{header}_distribution.png")
+      ).write("#{$export_dir}/Chart_#{stats_manager.name}_#{header}_distribution.png")
 
     when 'Event'
-      hist = stats.stats[index]
+      hist = stats.stats
       keys = hist.keys.sort{|a,b| a.to_i <=> b.to_i}
       values = keys.map{|k| hist[k]}
       Geoptima::Chart.draw_category_chart(
-        stats.name, keys, values,
+        stats_manager.name, keys, values,
         :title => "#{header} Distribution",
         :width => 1024
-      ).write("#{$export_dir}/Chart_#{stats.name}_#{header}_distribution.png")
+      ).write("#{$export_dir}/Chart_#{stats_manager.name}_#{header}_distribution.png")
 
     else
-      if stats.diverse?(index)
+      if stats.diverse?
         puts "Ignoring high diversity field #{header}"
       else
-        puts "Charting field: #{header} with length #{stats.length(index)} and diversity #{stats.diversity(index)}"
-        hist = stats.stats[index]
+        puts "Charting field: #{header} with length #{stats.length} and diversity #{stats.diversity}"
+        hist = stats.stats
         keys = hist.keys.sort{|a,b| a.to_i <=> b.to_i}
         values = keys.map{|k| hist[k]}
-        args = [stats.name, keys, values, {
+        args = [stats_manager.name, keys, values, {
           :title => "#{header} Distribution",
           :width => 1024}]
-        g = (stats.length(index) > 50) ?
+        g = (stats.length > 50) ?
               Geoptima::Chart.draw_line_chart(*args) :
-            (stats.length(index) > 10 || stats.numerical?(index)) ?
+            (stats.length > 10 || stats.numerical?) ?
               Geoptima::Chart.draw_histogram_chart(*args) :
-            (stats.length(index) > 1) ?
+            (stats.length > 1) ?
               Geoptima::Chart.draw_category_chart(*args) :
             nil
-        g && g.write("#{$export_dir}/Chart_#{stats.name}_#{header}_distribution.png")
+        g && g.write("#{$export_dir}/Chart_#{stats_manager.name}_#{header}_distribution.png")
       end
     end
   end
+end
+
+#
+# Now run the actual program, reading the specifation file and then the CSV files
+#
+
+$stats_managers = {}
+
+$specfile && $specs = Geoptima::StatsSpecs.new($specfile)
+
+$files.each do |file|
+  lines = 0
+  headers = nil
+  filename = File.basename(file)
+  (names = filename.split(/[_\.]/)).pop
+  name = $merge_all ? ($merged_name || 'All') : names.join('_')
+  $stats_managers[name] ||= StatsManager.new(name)
+  puts "About to read file #{file}"
+  File.open(file).each do |line|
+    lines += 1
+    fields=line.chomp.split(/\t/)
+    if headers
+      puts "Processing line: #{line}" if($debug)
+      $stats_managers[name].add_all(fields,headers)
+    else
+      headers = fields
+      if headers.length<2
+        puts "Too few headers, rejecting #{file}"
+        break
+      end
+      $stats_managers[name].set_headers(headers)
+    end
+  end
+end
+
+# Finally output all charts specified
+
+$stats_managers.each do |name,stats_manager|
+  if $specs
+    $specs.chart_specs.each do |chart_spec|
+      chart_spec.process(stats_manager)
+    end
+  end
+  if $create_all
+    create_all name, stats_manager
   end
 end
 

data/examples/show_geoptima.rb

@@ -7,7 +7,7 @@ $: << '../lib'
 require 'date'
 require 'geoptima'
 
-Geoptima::assert_version("0.0.9")
+Geoptima::assert_version("0.1.0")
 
 $debug=false
 

data/lib/geoptima/chart.rb

@@ -20,6 +20,12 @@ end
 
 module Geoptima
   class Chart
+    def self.libs
+      $chart_libs
+    end
+    def self.available?
+      $chart_libs.length>0
+    end
     DEFAULT_OPTIONS = {:show_points => true, :show_lines => true, :title => nil, :width => 800, :margins => 20, :font_size => 14}
     attr_reader :chart_type
     attr_accessor :chart, :data, :options
@@ -55,6 +61,18 @@ module Geoptima
       options[:filename] && g.write(options[:filename])
       g
     end
+    def self.draw_grouped_chart(legends,keys,values,options={})
+      puts "Creating a chart with legends #{legends.inspect} for #{keys.length} keys"
+      chart_type = (options[:chart_type]==:line) ? :line : :bar
+      g = make_chart(chart_type, options)
+      legends.each do |legend|
+        g.data(legend, values[legend])
+      end
+      g.minimum_value = 0
+      g.labels = keys.inject({}){|a,v| a[a.length] = v;a}
+      options[:filename] && g.write(options[:filename])
+      g
+    end
     def self.draw_category_chart(legend,keys,values,options={})
       puts "Creating category chart with keys: #{keys.join(',')}"
       puts "Creating category chart with values: #{values.join(',')}"

data/lib/geoptima/data.rb

@@ -2,21 +2,7 @@
 
 require 'rubygems'
 require 'multi_json'
-require 'date'
-if $use_dateperformance
-  begin
-    require 'date/performance'
-    require 'date/memoize'
-    class DateTime
-      def >(other) ; self - other > 0 ; end
-      def <(other) ; self - other < 0 ; end
-      def >=(other); self - other >= 0; end
-      def <=(other); self - other <= 0; end
-    end
-  rescue LoadError
-    puts "No date-performance gem installed, some features will run slower"
-  end
-end
+require 'geoptima/daterange'
 
 #
 # The Geoptima Module provides support for the Geoptima Client JSON file format
@@ -43,26 +29,6 @@ module Geoptima
     end
   end
 
-  class DateRange
-    attr_reader :min, :max, :range
-    def initialize(min,max)
-      @min = min
-      @max = max
-      @range = Range.new(min,max)
-    end
-    if ENV['RUBY_VERSION'] =~ /1\.8/
-      puts "Defining Range.include? to wrap for 1.8"
-      def include?(time)
-        @range.include?(time)
-      end
-    else
-      puts "Defining Range.include? to perform inequality tests for 1.9"
-      def include?(time)
-        (time >= min) && (time <= @max)
-      end
-    end
-  end
-
   # The Geoptima::Event class represents and individual record or event
   class Event
     KNOWN_HEADERS={

data/lib/geoptima/daterange.rb

@@ -0,0 +1,40 @@
+#!/usr/bin/env ruby
+
+require 'date'
+if $use_dateperformance
+  begin
+    require 'date/performance'
+    require 'date/memoize'
+    class DateTime
+      def >(other) ; self - other > 0 ; end
+      def <(other) ; self - other < 0 ; end
+      def >=(other); self - other >= 0; end
+      def <=(other); self - other <= 0; end
+    end
+  rescue LoadError
+    puts "No date-performance gem installed, some features will run slower"
+  end
+end
+
+module Geoptima
+  class DateRange
+    attr_reader :min, :max, :range
+    def initialize(min,max)
+      @min = min
+      @max = max
+      @range = Range.new(min,max)
+    end
+    if ENV['RUBY_VERSION'] =~ /1\.8/
+      puts "Defining Range.include? to wrap for 1.8"
+      def include?(time)
+        @range.include?(time)
+      end
+    else
+      puts "Defining Range.include? to perform inequality tests for 1.9"
+      def include?(time)
+        (time >= min) && (time <= @max)
+      end
+    end
+  end
+end
+

data/lib/geoptima/version.rb

@@ -1,6 +1,6 @@
 module Geoptima
 
-  VERSION = "0.0.9"
+  VERSION = "0.1.0"
 
   def self.version_as_int(ver)
     base = 1

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: geoptima
 version: !ruby/object:Gem::Version 
-  hash: 13
+  hash: 27
   prerelease: 
   segments: 
   - 0
+  - 1
   - 0
-  - 9
-  version: 0.0.9
+  version: 0.1.0
 platform: ruby
 authors: 
 - Craig Taverner
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-03-21 00:00:00 Z
+date: 2012-03-22 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   name: multi_json
@@ -78,6 +78,7 @@ files:
 - lib/geoptima/data.rb
 - lib/geoptima/chart.rb
 - lib/geoptima/options.rb
+- lib/geoptima/daterange.rb
 - lib/geoptima.rb
 - examples/show_geoptima_sos.rb
 - examples/show_geoptima.rb