geoptima 0.0.1

data/examples/show_geoptima.rb

@@ -0,0 +1,161 @@
+#!/usr/bin/env ruby
+
+# useful if being run inside a source code checkout
+$: << 'lib'
+$: << '../lib'
+
+require 'date'
+require 'geoptima'
+
+$debug=false
+
+$event_names=[]
+$files = []
+$print_limit = 10000
+
+def cw(val)
+  val.nil? ? '' : "(#{val})"
+end
+
+while arg=ARGV.shift:
+  if arg =~ /^\-(\w+)/
+    $1.split(//).each do |aa|
+      case aa
+      when 'h'
+        $help=true
+      when 'd'
+        $debug=true
+      when 'p'
+        $print=true
+      when 'v'
+        $verbose=true
+      when 'x'
+        $export=true
+      when 's'
+        $seperate=true
+      when 'E'
+        $event_names += ARGV.shift.split(/[\,\;\:\.]+/)
+      when 'T'
+        $time_range = Range.new(*(ARGV.shift.split(/[\,]+/).map do |t|
+          DateTime.parse t
+        end))
+      when 'L'
+        $print_limit = ARGV.shift.to_i
+        $print_limit = 10 if($print_limit<1)
+      else
+        puts "Unrecognized option: -#{aa}"
+      end
+    end
+  else
+    if File.exist? arg
+      $files << arg
+    else
+      puts "No such file: #{arg}"
+    end
+  end
+end
+
+$help = true if($files.length < 1)
+if $help
+  puts <<EOHELP
+Usage: ./showGeoptimaEvents.rb <-dvxEh> <-L limit> <-E types> <-T min,max> file <files>
+  -d  debug mode (output more context during processing) #{cw $debug}
+  -p  print mode (print out final results to console) #{cw $print}
+  -v  verbose mode (output extra information to console) #{cw $verbose}
+  -x  export IMEI specific CSV files for further processing #{cw $export}
+  -s  seperate the export files by event type #{cw $seperate}
+  -h  show this help
+  -L  limit verbose output to specific number of lines #{cw $print_limit}
+  -E  comma-seperated list of event types to show and export (default: all; current: #{$event_names.join(',')})
+  -T  time range to limit results to (default: all; current: #{$time_range})
+EOHELP
+  exit 0
+end
+
+$verbose = $verbose || $debug
+$datasets = Geoptima::Dataset.make_datasets($files, :locate => true, :time_range => $time_range)
+
+class Export
+  attr_reader :files, :names, :headers
+  def initialize(imei,names,dataset)
+    @names = names
+    if $export
+      if $seperate
+        @files = names.inject({}) do |a,name|
+          a[name] = File.open("#{imei}_#{name}.csv",'w')
+          a
+        end
+      else
+        @files={nil => File.open("#{imei}.csv",'w')}
+      end
+    end
+    @headers = names.inject({}) do |a,name|
+      a[name] = dataset.header([name]).reject{|h| h === 'timeoffset'}
+      puts "Created header for name #{name}: #{a[name].join(',')}" if($debug)
+      a
+    end
+    @headers[nil] = @headers.values.flatten
+    files && files.each do |key,file|
+      file.puts "Time\tEvent\tLatitude\tLongitude\t#{header(key).join("\t")}\n"
+    end
+    if $debug || $verbose
+      @headers.each do |name,head|
+        puts "Header[#{name}]: #{head.join(',')}"
+      end
+    end
+  end
+  def header(name=nil)
+    @headers[name]
+  end
+  def puts_to(line,name)
+    name = nil unless($seperate)
+    files[name].puts(line) if($export && files[name])
+  end
+  def puts_to_all(line)
+    files && files.each do |key,file|
+      file.puts line
+    end
+  end
+  def close
+    files && files.each do |key,file|
+      file.close
+      @files[key] = nil
+    end
+  end
+end
+
+def if_le
+  $count ||= 0
+  if $print
+    if $count < $print_limit
+      yield
+    elsif $count == $print_limit
+      puts " ... "
+    end
+  end
+  $count += 1
+end
+
+puts "Found #{$datasets.length} IMEIs"
+$datasets.keys.sort.each do |imei|
+  dataset = $datasets[imei]
+  events = dataset.sorted
+  puts if($print)
+  puts "Found #{dataset}"
+  if events && ($print || $export)
+    names = $event_names
+    names = dataset.events_names if(names.length<1)
+    export = Export.new(imei,names,dataset)
+    events.each do |event|
+      names.each do |name|
+        if event.name === name
+          fields = export.header($seperate ? name : nil).map{|h| event[h]}
+          export.puts_to "#{event.time_key}\t#{event.name}\t#{event.latitude}\t#{event.longitude}\t#{fields.join("\t")}", name
+          if_le{puts "#{event.time_key}\t#{event.name}\t#{event.latitude}\t#{event.longitude}\t#{event.fields.inspect}"}
+        end
+      end
+    end
+    export.close
+  end
+end
+

data/examples/show_geoptima_sos.rb

@@ -0,0 +1,50 @@
+#!/usr/bin/env ruby
+
+# useful if being run inside a source code checkout
+$: << 'lib'
+$: << '../lib'
+
+require 'date'
+require 'geoptima'
+
+$debug=false
+
+$events={}
+$files = []
+
+$events={}
+
+while arg=ARGV.shift:
+  if arg =~ /^\-(\w+)/
+    $1.split(//).each do |aa|
+      case aa
+      when 'd'
+        $debug=true
+      else
+        puts "Unrecognized option: -#{aa}"
+      end
+    end
+  else
+    if File.exist? arg
+      $files << arg
+    else
+      puts "No such file: #{arg}"
+    end
+  end
+end
+
+$datasets = Geoptima::Dataset.make_datasets($files, :locate => true)
+
+puts "Found #{$datasets.length} IMEIs"
+$datasets.keys.sort.each do |imei|
+  dataset = $datasets[imei]
+  events = dataset.sorted('mode')
+  puts "\nFor #{imei} found #{dataset.length} events:\n"
+  if events && events.length>0
+    puts "Time\tLat\tLon\t#{events.first.header[1..-1].join("\t")}"
+    events.each do |event|
+      puts "#{event.time}\t#{event.latitude}\t#{event.longitude}\t#{event.data[1..-1].join("\t")}"
+    end
+  end
+end
+

data/examples/stats.rb

@@ -0,0 +1,41 @@
+#!/usr/bin/env ruby
+
+$debug = false
+$lines = 0
+
+ARGF.each do |line|
+  $lines += 1
+  fields=line.chomp.split(/\t/)
+  if $headers
+    puts "Processing line: #{line}" if($debug)
+    fields.each_with_index do |field,index|
+      puts "\tField[#{index}]: #{field}" if($debug)
+      stats = $stats[index]
+      stats[field] ||= 0
+      stats[field] += 1
+      puts "\tField[#{index}]: #{field} => #{stats[field]}" if($debug)
+    end
+  else
+    $headers = fields
+    $stats = fields.map{|h| {}}
+  end
+end
+
+$headers.each_with_index do |header,index|
+  stats = $stats[index]
+  values = stats.keys
+  perc = 100.0 * values.length.to_f / $lines.to_f
+  puts "\nFound #{values.length} unique values for field '#{header}'"
+  if values.length > 500
+    puts "\tNot printing more values more diverse than 500"
+  elsif (perc > 75)
+    puts "\tNot printing more values more diverse than #{perc}%"
+  else
+    puts header
+    values.sort.each do |value|
+      value_text = (value.to_s.length < 1) ? '<empty>' : value
+      puts "\t#{value_text}\t#{stats[value]}"
+    end
+  end
+end
+

data/geoptima.gemspec

@@ -0,0 +1,31 @@
+lib = File.expand_path('../lib/', __FILE__)
+$:.unshift lib unless $:.include?(lib)
+ 
+require 'geoptima/version'
+
+Gem::Specification.new do |s|
+  s.name = "geoptima"
+  s.version = Geoptima::VERSION
+  s.authors = "Craig Taverner"
+  s.email = 'craig@amanzi.com'
+  s.homepage = "http://github.com/craigtaverner/geoptima.rb"
+  s.rubyforge_project = 'geoptima'
+  s.summary = "Ruby access to Geoptima JSON files"
+  s.description = <<-EOF
+Geoptima is a suite of applications for measuring and locating mobile/cellular subscriber experience on GPS enabled smartphones.
+It is produced by AmanziTel AB in Helsingborg, Sweden, and supports many phone manufacturers, with free downloads from the
+various app stores, markets or marketplaces. This Ruby library is only capable of reading the JSON format files priduced by these phones
+and reformating them as CSV for further analysis in Excel. This is a simple and independent way of analysing the data, when
+compared to the full-featured analysis applications and servers available from AmanziTel. If you want to analyse a limited amount
+of data in excel, or with Ruby, then this GEM might be for you. If you want to analyse large amounts of data, from many subscribers, or over long periods of time
+then rather consider the NetView and Customer IQ applications from AmanziTel at www.amanzitel.com.
+EOF
+
+  s.require_path = 'lib'
+  s.files        = Dir.glob("{bin,lib,examples}/**/*").reject{|x| x=~/(tmp|target|test-data)/} + %w(README.rdoc CHANGELOG CONTRIBUTORS Gemfile geoptima.gemspec)
+  s.executables  = ['show_geoptima']
+  s.has_rdoc = true
+  s.extra_rdoc_files = %w( README.rdoc )
+  s.rdoc_options = ["--quiet", "--title", "geoptima.rb", "--opname", "index.html", "--line-numbers", "--main", "README.rdoc", "--inline-source"]
+  s.required_ruby_version = ">= 1.8.7"
+end

data/lib/geoptima/data.rb

@@ -0,0 +1,311 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'json'
+require 'date'
+
+#
+# The Geoptima Module provides support for the Geoptima Client JSON file format
+#
+module Geoptima
+
+  SPERDAY = 60*60*24
+  MSPERDAY = 1000*60*60*24
+  SHORT = 256*256
+
+  class Config
+    DEFAULT={:min_datetime => DateTime.parse("2000-01-01"), :max_datetime => DateTime.parse("2030-01-01")}
+    def self.config(options={})
+      @@options = (@@options || DEFAULT).merge(options)
+    end
+    def self.options
+      @@options ||= DEFAULT
+    end
+    def self.[](key)
+      options[key]
+    end
+    def self.[]=(key,value)
+      options[key] = value
+    end
+  end
+
+  # The Geoptima::Event class represents and individual record or event
+  class Event
+    attr_reader :header, :name, :data, :fields, :time, :latitude, :longitude
+    def initialize(start,name,header,data)
+      @name = name
+      @header = header
+      @data = data
+      @fields = @header.inject({}) do |a,v|
+        a[v] = @data[a.length]
+        a
+      end
+      @time = start + (@fields['timeoffset'].to_f / MSPERDAY.to_f)
+      @fields.reject!{|k,v| k=~/timeoffset/}
+      if @fields['cell_id'].to_i > SHORT
+        @fields['cell_id'] = @fields['cell_id'].to_i % SHORT
+      end
+      puts "Created Event: #{self}" if($debug)
+    end
+    def utc
+      time.new_offset(0)
+    end
+    def time_key
+      utc.strftime("%Y-%m-%d %H:%M:%S.%3N")
+    end
+    def [](key)
+      @fields[key]
+    end
+    def -(other)
+      (self.time - other.time) * SPERDAY
+    end
+    def closer_than(other,seconds=60)
+      (self - other).abs < seconds
+    end
+    def locate(gps)
+      @latitude = gps['latitude']
+      @longitude = gps['longitude']
+    end
+    def locate_if_closer_than(gps,seconds=60)
+      locate(gps) if(closer_than(gps,seconds))
+    end
+    def to_s
+      "#{name}[#{time}]: #{@fields.inspect}"
+    end
+  end
+
+  # The Geoptima::Data is an entire JSON file of events
+  class Data
+    attr_reader :path, :json, :count
+    def initialize(path)
+      @path = path
+      @json = JSON.parse(File.read(path))
+      if $debug
+        puts "Read Geoptima: #{geoptima}"
+        puts "\tSubscriber: #{subscriber.to_json}"
+        puts "\tIMSI: #{imsi}"
+        puts "\tIMEI: #{imei}"
+        puts "\tStart: #{start}"
+      end
+    end
+    def to_s
+      json.to_json[0..100]
+    end
+    def geoptima
+      @geoptima ||= json['geoptima']
+    end
+    def subscriber
+      @subscriber ||= geoptima['subscriber']
+    end
+    def imsi
+      @imsi ||= subscriber['imsi']
+    end
+    def imei
+      @imei ||= subscriber['imei']
+    end
+    def platform
+      @platform ||= subscriber['Platform'] || subscriber['platform']
+    end
+    def model
+      @model ||= subscriber['Model'] || subscriber['model']
+    end
+    def os
+      @os ||= subscriber['OS']
+    end
+    def start
+      @start ||= subscriber['start'] && DateTime.parse(subscriber['start'].gsub(/Asia\/Bangkok/,'GMT+7').gsub(/Mar 17 2044/,'Feb 14 2012'))
+    end
+    def valid?
+      start && start > Data.min_start && start < Data.max_start
+    end
+    def self.min_start
+      @@min_start ||= DateTime.parse("2010-01-01 00:00:00")
+    end
+    def self.max_start
+      @@max_start ||= DateTime.parse("2020-01-01 00:00:00")
+    end
+    def events
+      @events ||= make_events
+    end
+    def events_names
+      events.keys.sort
+    end
+    def make_hash(name)
+      geoptima[name].inject({}) do |a,md|
+        key = md.keys[0]
+        a[key]=md[key]
+        a
+      end
+    end
+    def make_events
+      @count = 0
+      @events_metadata = make_hash('events-metadata')
+      events_data = {}
+      geoptima['events'].each do |data|
+        events = data['values']
+        event_type = data.keys.reject{|k| k=~/values/}[0]
+        header = @events_metadata[event_type]
+        if header
+          events_data[event_type] = (0...data[event_type].to_i).inject([]) do |a,block|
+            index = header.length * block
+            data = events[index...(index+header.length)]
+            if data && data.length == header.length
+              @count += 1
+              a << Event.new(start,event_type,header,data)
+            else
+              puts "Invalid '#{event_type}' data block #{block}: #{data.inspect}"
+              break a
+            end
+          end
+          if $debug
+            puts "Have '#{event_type}' event data:"
+            puts "\t#{header.join("\t")}"
+            events_data[event_type].each do |d|
+              puts "\t#{d.data.join("\t")}"
+            end
+          end
+        else
+          puts "No header found for event type: #{event_type}"
+        end
+      end
+      events_data
+    end
+  end
+
+  class Dataset
+
+    attr_reader :imei, :options
+
+    def initialize(imei,options={})
+      @imei = imei
+      @data = []
+      @options = options
+      @time_range = options[:time_range] || Range.new(Config[:min_datetime],Config[:max_datetime])
+    end
+
+    def <<(data)
+      @sorted = nil
+      @data << data
+    end
+
+    def file_count
+      @data.length
+    end
+
+    def imsi
+      imsis[0]
+    end
+
+    def imsis
+      @imsis ||= @data.inject({}) do |a,d|
+        a[d.imsi] ||= 0
+        a[d.imsi] += d.count.to_i
+        a
+      end.to_a.sort do |a,b|
+        b[1]<=>a[1]
+      end.map do |x|
+        #puts "Have IMSI: #{x.join('=')}"
+        x[0]
+      end.compact.uniq
+    end
+
+    def platform
+      @platform ||= @data.map{|d| d.platform}.compact.uniq[0]
+    end
+
+    def model
+      @model ||= @data.map{|d| d.model}.compact.uniq[0]
+    end
+
+    def os
+      @os ||= @data.map{|d| d.os}.compact.uniq[0]
+    end
+
+    def first
+      merge_events unless @sorted
+      @sorted[nil][0]
+    end
+
+    def last
+      merge_events unless @sorted
+      @sorted[nil][-1]
+    end
+
+    def length
+      sorted.length
+    end
+
+    def sorted(event_type=nil)
+      merge_events unless @sorted
+      unless @sorted[event_type] || event_type.nil?
+        @sorted[event_type] = @sorted[nil].reject do |event|
+          event.name != event_type
+        end
+      end
+      @sorted[event_type]
+    end
+
+    def header(names=nil)
+      merge_events unless @sorted
+      (names || events_names).map do |name|
+        [(s=sorted(name)[0]) && s.header]
+      end.flatten
+    end
+
+    def events_names
+      @data.map{ |v| v.events_names }.flatten.uniq.sort
+    end
+
+    def merge_events
+      @sorted ||= {}
+      unless @sorted[nil]
+        event_hash = {}
+        events_names.each do |name|
+          @data.each do |data|
+            (events = data.events[name]) && events.each do |event|
+              if @time_range.include?(event.time)
+                key = "#{event.time_key} #{name}"
+                event_hash[key] = event
+              end
+            end
+          end
+        end
+        @sorted[nil] = event_hash.keys.sort.map{|k| event_hash[k]}
+        locate_events if(options[:locate])
+      end
+      @sorted
+    end
+
+    def locate_events
+      prev_gps = nil
+      sorted.each do |event|
+        if event.name === 'gps'
+          prev_gps = event
+        elsif prev_gps
+          event.locate_if_closer_than(prev_gps,60)
+        end
+      end
+    end
+
+    def to_s
+      "IMEI:#{imei}, IMSI:#{imsis.join(',')}, Platform:#{platform}, Model:#{model}, OS:#{os}, Files:#{file_count}, Events:#{sorted.length}"
+    end
+
+    def self.make_datasets(files, options={})
+      datasets = {}
+      files.each do |file|
+        geoptima=Geoptima::Data.new(file)
+        unless geoptima.valid?
+          puts "INVALID: #{geoptima.start}\t#{file}\n\n"
+        else
+          datasets[geoptima.imei] ||= Geoptima::Dataset.new(geoptima.imei, options)
+          datasets[geoptima.imei] << geoptima
+        end
+      end
+      datasets
+    end
+
+  end
+
+end
+

data/lib/geoptima/version.rb

@@ -0,0 +1,3 @@
+module Geoptima
+  VERSION = "0.0.1"
+end

data/lib/geoptima.rb

@@ -0,0 +1,3 @@
+require 'geoptima/version.rb'
+require 'geoptima/data.rb'
+

metadata

@@ -0,0 +1,96 @@
+--- !ruby/object:Gem::Specification 
+name: geoptima
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: 
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- Craig Taverner
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-03-01 00:00:00 Z
+dependencies: []
+
+description: |
+  Geoptima is a suite of applications for measuring and locating mobile/cellular subscriber experience on GPS enabled smartphones.
+  It is produced by AmanziTel AB in Helsingborg, Sweden, and supports many phone manufacturers, with free downloads from the
+  various app stores, markets or marketplaces. This Ruby library is only capable of reading the JSON format files priduced by these phones
+  and reformating them as CSV for further analysis in Excel. This is a simple and independent way of analysing the data, when
+  compared to the full-featured analysis applications and servers available from AmanziTel. If you want to analyse a limited amount
+  of data in excel, or with Ruby, then this GEM might be for you. If you want to analyse large amounts of data, from many subscribers, or over long periods of time
+  then rather consider the NetView and Customer IQ applications from AmanziTel at www.amanzitel.com.
+
+email: craig@amanzi.com
+executables: 
+- show_geoptima
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+files: 
+- bin/show_geoptima
+- lib/geoptima/version.rb
+- lib/geoptima/data.rb
+- lib/geoptima.rb
+- examples/show_geoptima_sos.rb
+- examples/export_layer.rb
+- examples/stats.rb
+- examples/show_geoptima.rb
+- examples/sample_geoptima.json
+- README.rdoc
+- CHANGELOG
+- CONTRIBUTORS
+- Gemfile
+- geoptima.gemspec
+homepage: http://github.com/craigtaverner/geoptima.rb
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --quiet
+- --title
+- geoptima.rb
+- --opname
+- index.html
+- --line-numbers
+- --main
+- README.rdoc
+- --inline-source
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 57
+      segments: 
+      - 1
+      - 8
+      - 7
+      version: 1.8.7
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: geoptima
+rubygems_version: 1.7.2
+signing_key: 
+specification_version: 3
+summary: Ruby access to Geoptima JSON files
+test_files: []
+