primord-tools 1.0.0 → 1.0.3

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    MTdiZjAzZGNjOGYxNDJiNTY1OWFjNWFmOTdmMDhlNDk1NmQ5ZTQwZg==
+    YWVjZTlhNjQ4MDgyMjIxY2I5NWE3N2ZkNjIwZGVmZThhOGFjYjAzMQ==
   data.tar.gz: !binary |-
-    OTdjYjY1YzNlZjg5YjM4ZmM1YjYzOTI0NGNkYmIyYmUyNzMzYWU2Nw==
+    MTMzZWIyMzRhYTVkYzFlMTAyZGU5ZWYyYWRhOGQ2MDcxZDBhN2U2NQ==
 SHA512:
   metadata.gz: !binary |-
-    MjM4NWJmMGE3N2M1ODQ2MmExMmIwY2E1YjVjODdiYzZiN2E0NTg4Y2ZjZTA5
-    ZjQ2NjNhNzU0NGU1OWQ3NmM2ZTY5ZTU5YjAzZWY0OTNhZWI1ZjdiOGY0ZDY5
-    ODI2YTU2MzEyNjQ0NWMzMDc1NzBlMDkyMzk0NjZlMDAyMTA2NjY=
+    NWZiMmU4OTUzMmU2OTM1NDcyMWFiODg4YWNjM2E2NjY1YzAzNTU2Njk4MGFl
+    OTUwZWJjZjg4NTc4YTZjOTYxZDRmOTJiOGEyOTQ1YmY4YTY2ZDY2YTI1Y2Rj
+    YmJiMGU5ZmEwZDBkOTdhNGEyZDlkNjE0YTM5Njg4NjQxODg2OWM=
   data.tar.gz: !binary |-
-    YzNkZjI4YWIyOTk2MTMxY2I0NzdlYTk2NzVjZGRiZjhhNzBmNjU1YWVlOTNj
-    MmE1MzRlYjBlOWIwYTc0MTA4OTdlN2E3NWE2OWU1ZGM2M2M5ZDA3YzBlNDNh
-    ZGZiOGEzOTY0YTA3NGNlYzEyZjBhYjUzYzZhZTdjZWUwODI4ZTY=
+    MmI3NmQ2MTVmN2RkMzA5YTdmOWEwY2UyZDRjMzEwMDY0ZDcwYmJjZWU4ODcx
+    YWY3MDMzNzFjMTE0NWYyNzU2N2QzNTM3MWZmODAxNjg5ZTU2YmI1ZmFmZTQ2
+    YzcwNWFlYzY4NDAyYjBjMmM5MjhkMjI4YjZlZDYxNTk0ZTkyNzI=

data/.gitignore

@@ -1 +1,5 @@
 /pkg
+
+# that is a symlink to the actual data
+/stuff
+.R*

data/README.md

@@ -2,3 +2,65 @@ primord-tools
 =============
 
 Tools used to create metadata of primord rf data
+
+Desinged to analyze the three types of files generated
+by the experiment: bgdist, pylog, and pyerr.
+
+Usage
+-----
+
+The basic usage is `primord-metadata <directive> ...`
+
+Supported directives are
+ * `metadata` - generate metadata
+ * `help` - print help message
+
+Metadata
+--------
+
+The syntax for this directive is `primord-metadata metadata <outfile> <bgdist|pyerr|pylog> <param,param,...> <files>`
+
+ * `outfile` - file to write the output to in csv format
+ * `bgdist|pyerr|pylog` - the type of file to process (only one can be specified)
+ * `param,param,...` - the parameters to collect/compute (eg. `file,lines,date` for the filename, number of lines and date)
+ * `files` - a space seperated list of input files
+
+
+
+Parametes
+---------
+
+All files support these parameters
+ * `file` - name of the file
+ * `date` - date of data gathered
+ * `lines` - number of lines in the file
+
+The following parameters are avalible for `bgdist`
+ * `has_mulit` - whether the data has multi-frequency data
+ * `stat_sf_aic` - the Akaike Information Criterion for the single frequency model
+ * `stat_sf_intercept` - the count intercept used in the single frequency model
+ * `stat_mf_aic` - the Akaike Information Criterion for the single frequency model
+ * `stat_mf_intercept` - the count intercept used in the multi frequency model
+ * `stat_sf_count` - number of single frequency events observed
+ * `stat_mf_count` - number of multi frequency events observed
+
+The following parameters are avalible for `pylog`
+
+The following parameters are avalible for `pyerr`
+
+
+BGDist Model
+------------
+
+The background noise should follow a poisson distribution. This is no suprise, as
+many random processes follow this distribution. Due to hardware limitations of the
+windowed FFT as well as bandwidth limitations between the USRP2 and the computer
+logging the data, signals below a threshold were not logged. This makes computing
+the poisson PDF for the data slightly more chalenging. The method that is employed
+is a zero inflated GLM using a logarithmic link function. 
+
+In many cases the model fails due to the following reasons
+ * insuficient data - if there is only signals in a couple of bins the optimization fails
+ * other wierd/unkonwn issues - I am investegating other sourced of failures
+
+

data/Rakefile

@@ -13,7 +13,7 @@ spec = Gem::Specification.new do |gem|
     gem.summary      = "Creates metadata for primord rf data"
     gem.description  = "Process the logs from gnuradio and generates metadata"
     
-    gem.files        = `git ls-files`.split($\)
+    gem.files        = `git ls-files | grep -E '^tmp' -v`.split($\)
     
     gem.executables  = ["primord-metadata"]
 end

data/TODO.md

@@ -0,0 +1,25 @@
+Main part
+ * a good README
+ * a better help message
+ * option parsing
+ * pyerr/pylog - parse the logs and find errors
+ * cross coralate time of error to bgdist data
+ * bgdist - get the threshold
+ * bgdist - aggragate data in a short time periods together
+ * bgdist - find data duration
+
+On the R side of things 
+ * fit poisson to data [here](http://stats.stackexchange.com/questions/70558/diagnostic-plots-for-count-regression)
+   and [here](http://www.ats.ucla.edu/stat/r/dae/zipoisson.htm)
+ * get it working on windows
+ * caching of computed statistics to make re-running it on largish subsets of the data faster
+ * bgdist - get mu and lambda
+
+Package Issues
+ * Fix up the rake file
+ * rinruby is a pain so do something about it
+
+Wishlist
+ * get data in a time line
+ * make a nice web ui with an interactive timeline
+ * make the timeline pinable

data/bin/primord-metadata

@@ -1,101 +1,207 @@
 #!/usr/bin/env ruby
 
-# TODO write to output file
-# TODO tabs vs space consistency
-# TODO other useful things
+# TODO proper option parsing
+# TODO this thing is turning into a beast
+#  - split into multiple files
+#  - modulize the code 
 
-module Utils
-    def self.print_usage
-        puts "./metadaer.rb <pyerr|pylog|bgdist> <param,param,...> <files>"
-    end
-    def self.count_lines file
-        lines = 0
-        file.each_line { |l| lines += 1 }
-        file.rewind
-        lines
-    end
-    def self.extract_date str
-        a = str.match /.*([0-9]{4})([0-9]{2})([0-9]{2})/
-        "#{a[2]}-#{a[3]}-#{a[1]}"
-    end
-end # class Stuff
+# Wonky bug in rinruby (see http://hfeild-software.blogspot.com/2013/01/rinruby-woes.html)
+# I will send a PR to get it fixed sometime in the future
+R = ""
+
+# TODO fix this in the Rakefile
+$:.unshift File.dirname(__FILE__) + '/../lib/'
+require 'utils'
 
-# classes for each type of file we want to
-# work with
+# classes for each type of file we want to work with
 class DataFile
     @@params
+    @@outfile
     def initialize filename
+        @filename = filename
         @data = {}
-        @data['file'] = filename
-        @file = File.open filename, 'r'
-
+        @lines = IO.readlines @filename
+        
+        # generic stuff that all files can have
+        def self.data_file
+            @data['file'] = @filename
+        end
+        def self.data_lines
+            @data['lines'] = @lines.length
+        end
+        def self.data_date
+            @data['date'] = Utils::extract_date @filename
+        end
+        
+        # all the computation happens here
         def self.write
+            # fill out @data
+            @@params.each do |param|
+                self.send "data_#{param}"
+            end
+            # write everything out to stdout
             str = ""
             @@params.each do |param|
                 str += "#{@data[param]},"
             end
-            puts str[0..-2]
+            @@outfile.puts str[0..-2]
         end
     end
-    def self.format= params
+    # set @@params and write the csv header
+    def self.format params, outfile
+        @@outfile = outfile
         @@params = params
         str = ""
         @@params.each do |param|
             str += "#{param},"
         end
-        puts str[0..-2]
+        @@outfile.puts str[0..-2]
+
+        # check if we need to do the statistical modeling
+        @@do_stats = false
+        @@params.each do |param|
+            if param =~ /^stat_/
+                @@do_stats = true
+                require 'rinruby'
+                # NOTE not even close to threadsafe
+                @@R = RinRuby.new :interactive => false, :echo => true, :executable => "'#{`which R`.chomp.chomp}'"
+                @@R.eval 'library(pscl)'
+                break
+            end
+        end
     end
 end
 
 class Pyerr < DataFile
     def initialize filename
         super
-        # count the lines
-        @data['lines'] = Utils::count_lines @file
-        # get the date		
-        @data['date'] = Utils::extract_date filename
+        # nothing special
     end
 end # class DataFile
 
 class Pylog < DataFile
     def initialize filename
         super
-        # count the lines
-        @data['lines'] = Utils::count_lines @file
-        # get the date		
-        @data['date'] = Utils::extract_date filename
+        # nothing special
     end
 end # class Pylog
 
 class Bgdist < DataFile
     def initialize filename
         super
-        # count the lines
-        @data['lines'] = Utils::count_lines @file
-        # get the date		
-        @data['date'] = Utils::extract_date filename
+        def self.data_has_multi
+            if @lines[0] =~ /bin,avg_amp,N_avg_amp,,freq,N_freq/
+                @data['has_multi'] = false
+            elsif @lines[0] =~ /bin,avg_amp_sf,N_avg_amp_sf,N_avg_amp_mf,,freq,N_freq_sf,N_freq_mf/
+                @data['has_multi'] = true
+            else
+                abort "\nerror in comprehending bgdist header: #{@lines[0]}\n"
+            end
+        end
+        if @@do_stats
+rcode = <<EOF
+    print_log <- function (what, msg)
+    {
+        print(paste("#{@filename}:", what, " -> ", msg), stdout());
+    }
+    
+    D <- read.csv('#{@filename}')
+    D <- D[1:length(D$bin)-1,]
+EOF
+            self.data_has_multi
+            if @data['has_multi']
+rcode += <<EOF
+    mod.sf <- tryCatch({
+        zeroinfl(D$N_avg_amp_sf ~ bin, data=D, dist="poisson")
+    }, warning = function(w) {
+        print_log("fitting single frequency", w)
+    }, error = function(e) {
+        print_log("fitting single frequency", e)
+    }, finally = {})
+    mod.sf_count = sum(D$N_avg_amp_sf)
+    
+    mod.mf <- tryCatch({
+        zeroinfl(D$N_avg_amp_mf ~ bin, data=D, dist="poisson")
+    }, warning = function(w) {
+        print_log("fitting multi frequency", w)
+    }, error = function(e) {
+        print_log("fitting multi frequency", e)
+    }, finally = {})
+    mod.mf_count = sum(D$N_avg_amp_mf)
+EOF
+            else # single frequency
+rcode += <<EOF
+    mod.sf <- tryCatch({
+        zeroinfl(D$N_avg_amp ~ bin, data=D, dist="poisson")
+    }, warning = function(w) {
+        print_log("fitting single frequency", w)
+    }, error = function(e) {
+        print_log("fitting single frequency", e)
+    }, finally = {})
+    mod.sf_count = sum(D$N_avg_amp)
+    
+    mod.mf = NA
+    mod.mf_count = as.integer(0)
+EOF
+            end # end of single or multifrequency block
+            # rinruby does not support the logical type, thus we use NaN which is a double
+rcode += <<EOF
+    if (typeof(mod.sf) == "list") {
+        mod.sf_aic = as.double(AIC(mod.sf))
+        mod.sf_intercept = mod.sf$coefficients$count[1]
+    } else {
+        mod.sf_aic = NaN
+        mod.sf_intercept = NaN
+    }
+    if (typeof(mod.mf) == "list") {
+        mod.mf_aic = as.double(AIC(mod.mf))
+        mod.mf_intercept = mod.mf$coefficients$count[1]
+    } else {
+        mod.mf_aic = NaN
+        mod.mf_intercept = NaN
+    }
+EOF
+            @@R.eval rcode
+        end
+        # ruby metaprograming :P
+        def self.method_missing m, *args, &block
+            if m =~ /^data_(stat_)(.*)/
+                @data["#{$1}#{$2}"] = @@R.send "mod.#{$2}"
+                if @data["#{$1}#{$2}"].respond_to? :nan? and @data["#{$1}#{$2}"].nan?
+                    @data["#{$1}#{$2}"] = 'NA'
+                end
+            else
+                abort "your missing your method named #{m}, you should go find it\n"
+            end
+        end
     end
 end # class BGdist
 
-# main
 
+
+# main
 directive = ARGV.shift
 case directive
 when 'help'
     Utils::print_usage
     exit 0
-else
+when 'metadata'
+    outfile = ARGV.shift
+    clas = Object.const_get ARGV.shift.capitalize
     params = []
     ARGV.shift.gsub(/([^,]+)/){ |p| params << p }
-    DataFile::format = params
-    clas = Object.const_get directive.capitalize
+    DataFile::format params, (File.open outfile, 'w')
+# This is the supper fancy option that I will work on this weekend
+# I want to create a one page web app that is centered on a timeline
+# 
+when 'timeserve'
+    
+    
+else
+    puts 'unknown directive'
 end
 
-objs = []
 ARGV.each do |filename|
-    objs << (clas.new filename)
-end
-
-objs.each do |obj|
+    obj = clas.new filename
     obj.write
 end

data/lib/utils.rb

@@ -0,0 +1,10 @@
+module Utils
+    def self.print_usage
+        puts "primord-metadata <directive> ...\n\tSee https://github.com/floomby/primord-tools for more info"
+    end
+    # get date the file was created from the filename
+    def self.extract_date str
+        a = str.match /.*([0-9]{4})([0-9]{2})([0-9]{2})/
+        "#{a[2]}-#{a[3]}-#{a[1]}"
+    end
+end # module Utils

data/setup-env.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# setup the environment needed to use R on windows
+# under cygwin with the rinruby gem
+
+export PATH=$PATH:/cygdrive/c/Program\ Files/R/R-2.15.2/bin/x64

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: primord-tools
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.3
 platform: ruby
 authors:
 - ! 'floomby
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-18 00:00:00.000000000 Z
+date: 2014-07-03 00:00:00.000000000 Z
 dependencies: []
 description: Process the logs from gnuradio and generates metadata
 email: ! 'floomby@nmt.edu
@@ -24,8 +24,10 @@ files:
 - .gitignore
 - README.md
 - Rakefile
-- TODO
+- TODO.md
 - bin/primord-metadata
+- lib/utils.rb
+- setup-env.sh
 homepage: https://github.com/floomby/gitver
 licenses: []
 metadata: {}

data/TODO

@@ -1,2 +0,0 @@
- 1) a good README
- 2) a better help message