optimus-ep 0.6.91 → 0.8.0

data/CHANGELOG

@@ -0,0 +1,13 @@
+v.0.8.0 Compatibility Warning: Lots of stuff has changed with the way column definition strings are computed. Test your existing scripts after updating! Architecture changes: Completely rewrote parser. String-based expressions now deprecate lambdas, though they'll continue to work in the future. This change fixes lots of odd bugs in defining computed columns and filters. Bug fixes: The --columns and --filter-columns options for eprime2tabfile now work.
+
+v0.6.9.1 Bug fix: Fix for parsing eprime log files in which leaf frames don't occur at the highest log level
+
+v0.6.9. New features: Supports parsing raw tab-delmimted files; Allows Procs as starting values for ComputedColumns. Bug fixes: No longer jumbles the order of Eprime log files
+
+v0.6.5. New features: Speed! Speed! Speed! Should be something like 10x faster. Procs are now allowed in ComputedColumns -- so if you want to do really arbitrary things in your column computation, have at it.
+
+v0.6.0. New features: Added extract_timings, a script to pull stimulus timing data from eprime files; Added a GenericRunner -- a class that greatly eases end-to-end transformation of eprime data. See extract_timings for an example of its use.
+
+v0.5.5. New features: Supports making multiple passes through data, to allow extracting multiple stimuli that occur on the same row; Created a general-purpose stimulus extractor to pull data out in the format: stimulus \t onset__time \t offset__time. Architectural changes: Created a BasicTransformer that performs all of the basic operations to datasets: column calculation, row filtering, and multipass; Moved ColumnCalculator, RowFilter, and Multipasser into a Transformers module; Support release via hoe / rubygems
+
+v0.5. First public gem release; Supports converting general Eprime files into tab-delimited files; Supports converting Eprime files into AFNI stim_times files

data/{License.txt → LICENSE}

@@ -1,5 +1,5 @@
 Optimus: Libraries and utilities for interacting with E-Prime data files
-Copyright (C) 2008 Board of Regents of the University of Wisconsin System
+Copyright (C) 2008-09 Board of Regents of the University of Wisconsin System
 
 This program is free software; you can redistribute it and/or
 modify it under the terms of version 2 of the GNU General Public

data/{Manifest.txt → Manifest}

@@ -1,20 +1,25 @@
+CHANGELOG
 GPL.txt
-History.txt
-License.txt
-Manifest.txt
-README.txt
+LICENSE
+Manifest
+README
 Rakefile
+autotest/discover.rb
 bin/eprime2tabfile
 bin/extract_timings
-lib/calculator.rb
-lib/eprime.rb
-lib/eprime_data.rb
-lib/eprime_reader.rb
 lib/eprimetab_parser.rb
 lib/excel_parser.rb
+lib/expression_parser/evaluators.rb
+lib/expression_parser/expressions.rb
 lib/log_file_parser.rb
+lib/optimus.rb
+lib/optimus_data.rb
+lib/optimus_reader.rb
+lib/parsed_calculator.rb
 lib/raw_tab_parser.rb
 lib/runners/generic_runner.rb
+lib/runners/yaml_template/option_parser.rb
+lib/runners/yaml_template/runner.rb
 lib/tabfile_parser.rb
 lib/tabfile_writer.rb
 lib/transformers/basic_transformer.rb
@@ -24,18 +29,25 @@ lib/transformers/row_filter.rb
 lib/transformers/timing_extractor.rb
 lib/version.rb
 lib/writers/stimtimes_writer.rb
-spec/calculator_spec.rb
-spec/eprime_data_spec.rb
-spec/eprime_reader_spec.rb
 spec/eprimetab_parser_spec.rb
 spec/excel_parser_spec.rb
+spec/expression_parser/evaluators_spec.rb
+spec/expression_parser/expressions_spec.rb
 spec/log_file_parser_spec.rb
+spec/optimus_data_spec.rb
+spec/optimus_reader_spec.rb
+spec/parsed_calculator_spec.rb
+spec/raw_tab_parser_spec.rb
 spec/runners/generic_runner_spec.rb
+spec/runners/yaml_template/option_parser_spec.rb
+spec/runners/yaml_template/runner_spec.rb
 spec/samples/bad_excel_tsv.txt
 spec/samples/corrupt_log_file.txt
 spec/samples/eprime_tsv.txt
 spec/samples/excel_tsv.txt
 spec/samples/optimus_log.txt
+spec/samples/optimus_log_utf16le.txt
+spec/samples/raw_tsv.txt
 spec/samples/short_columns.txt
 spec/samples/sorted_columns.txt
 spec/samples/std_columns.txt

data/{README.txt → README}

@@ -27,7 +27,7 @@ FIX (describe your package)
 (The GNU Public License, Version 2)
 
 Optimus: Libraries and utilities for interacting with E-Prime data files
-Copyright (C) 2008 Board of Regents of the University of Wisconsin System
+Copyright (C) 2008-09 Board of Regents of the University of Wisconsin System
 
 This program is free software; you can redistribute it and/or
 modify it under the terms of version 2 of the GNU General Public

data/Rakefile

@@ -1,12 +1,11 @@
-require 'rake'
-require 'spec/rake/spectask'
-#require 'config/requirements'
-require 'hoe'
-require 'config/hoe' # setup Hoe + all gem configuration
+require 'echoe'
+require 'lib/version'
 
-Dir['tasks/**/*.rake'].each { |rake| load rake }
-
-desc "Run specs"
-Spec::Rake::SpecTask.new('default') do |t|
-  t.spec_files = FileList['spec/**/*.rb']
+Echoe.new("optimus-ep", Optimus::VERSION::STRING) do |p|
+  p.author = "Nate Vack"
+  p.email = "njvack@wisc.edu"
+  p.summary = "Utilities to process behavioral data generated by E-Prime"
+  p.url = "http://github.com/njvack/optimus-ep"
+  p.runtime_dependencies = ["rparsec >=1.0"]
+  p.development_dependencies = ["rspec"]
 end

data/autotest/discover.rb

@@ -0,0 +1 @@
+Autotest.add_discovery { "rspec2" }

data/bin/eprime2tabfile

@@ -2,7 +2,7 @@
 
 # Part of the Optimus package for managing E-Prime data
 # 
-# Copyright (C) 2008 Board of Regents of the University of Wisconsin System
+# Copyright (C) 2008-09 Board of Regents of the University of Wisconsin System
 # 
 # Written by Nathan Vack <njvack@wisc.edu>, at the Waisman Laborotory for Brain
 # Imaging and Behavior, University of Wisconsin - Madison
@@ -10,9 +10,9 @@
 require 'rubygems'
 require 'optparse'
 gem 'optimus-ep'
-require 'eprime'
+require 'optimus'
 
-module Eprime
+module Optimus
   class LogToTabConverter
     attr_reader :options
     attr_accessor :data_out
@@ -38,12 +38,12 @@ module Eprime
     end
     
     def convert
-      eprime = Eprime::Data.new(@columns, :ignore_warnings => @options[:force])
+      optimus = Optimus::Data.new(@columns, :ignore_warnings => @options[:force])
       @options[:files].each do |filename|
         begin
           File.open(filename, 'r') do |f|
-            reader = Eprime::Reader.new(f, {:columns => @columns, :force => @options[:force]})
-            eprime.merge!(reader.eprime_data)
+            reader = Optimus::Reader.new(f, {:columns => @columns, :force => @options[:force]})
+            optimus.merge!(reader.optimus_data)
           end
         rescue Exception => e
           @err.puts e.message
@@ -51,7 +51,7 @@ module Eprime
           exit(1)
         end
       end
-      writer = Eprime::TabfileWriter.new(eprime, data_out, {
+      writer = Optimus::TabfileWriter.new(optimus, data_out, {
         :write_top_line => @options[:add_filename_line],
         :columns => @columns
         })
@@ -78,7 +78,7 @@ module Eprime
     def option_parser(option_hash, args)
       opts = OptionParser.new
 
-      opts.banner = "Usage: eprimelog2tabfile [options] INPUT_FILES"
+      opts.banner = "Usage: optimuslog2tabfile [options] INPUT_FILES"
       opts.separator ""
 
       opts.on('-o', '--outfile=OUTFILE', String,
@@ -143,9 +143,9 @@ module Eprime
 end
 
 begin
-  conv = Eprime::LogToTabConverter.new(ARGV)
+  conv = Optimus::LogToTabConverter.new(ARGV)
 rescue Exception => e
-  STDERR.puts e.message+"\nrun eprimelog2tabfile --help for options\n"
+  STDERR.puts e.message+"\nrun optimuslog2tabfile --help for options\n"
   exit(1)
 end
 

data/bin/extract_timings

@@ -2,18 +2,18 @@
 
 # Part of the Optimus package for managing E-Prime data
 # 
-# Copyright (C) 2008 Board of Regents of the University of Wisconsin System
+# Copyright (C) 2008-09 Board of Regents of the University of Wisconsin System
 # 
 # Written by Nathan Vack <njvack@wisc.edu>, at the Waisman Laborotory for Brain
 # Imaging and Behavior, University of Wisconsin - Madison
 
 require 'rubygems'
 require 'optparse'
-gem 'optimus-ep'
-require 'runners/generic_runner'
+#gem 'optimus-ep'
+require 'lib/runners/generic_runner'
 
 begin
-  txr = Eprime::Runners::GenericRunner.new(Eprime::Transformers::TimingExtractor, ARGV)
+  txr = Optimus::Runners::GenericRunner.new(Optimus::Transformers::TimingExtractor, ARGV)
   txr.process!
 rescue ArgumentError => e
   STDERR.puts e.message

data/lib/eprimetab_parser.rb

@@ -1,6 +1,6 @@
 # Part of the Optimus package for managing E-Prime data
 # 
-# Copyright (C) 2008 Board of Regents of the University of Wisconsin System
+# Copyright (C) 2008-09 Board of Regents of the University of Wisconsin System
 # 
 # Written by Nathan Vack <njvack@wisc.edu>, at the Waisman Laborotory for Brain
 # Imaging and Behavior, University of Wisconsin - Madison
@@ -9,9 +9,9 @@
 
 require 'tabfile_parser'
 
-module Eprime
+module Optimus
   class Reader
-    class EprimetabParser < TabfileParser
+    class OptimustabParser < TabfileParser
       def initialize(file, options = {})
         options = options.merge(:skip_lines => 3)
         super(file, options)

data/lib/excel_parser.rb

@@ -1,6 +1,6 @@
 # Part of the Optimus package for managing E-Prime data
 # 
-# Copyright (C) 2008 Board of Regents of the University of Wisconsin System
+# Copyright (C) 2008-09 Board of Regents of the University of Wisconsin System
 # 
 # Written by Nathan Vack <njvack@wisc.edu>, at the Waisman Laborotory for Brain
 # Imaging and Behavior, University of Wisconsin - Madison
@@ -9,7 +9,7 @@
 
 require 'tabfile_parser'
 
-module Eprime
+module Optimus
   class Reader
     class ExcelParser < TabfileParser
       def initialize(file, options = {})

data/lib/expression_parser/evaluators.rb

@@ -0,0 +1,188 @@
+# Part of the Optimus package for managing E-Prime data
+# 
+# Copyright (C) 2008-09 Board of Regents of the University of Wisconsin System
+# 
+# Written by Nathan Vack <njvack@wisc.edu>, at the Waisman Laborotory for Brain
+# Imaging and Behavior, University of Wisconsin - Madison
+#
+# Functions that actually handle evaluating operands and such.
+
+module Optimus
+  module ParsedCalculator
+    module Evaluators
+      NaN = 0.0/0.0
+      
+      class ArgList
+        include Enumerable
+        attr_accessor :args
+        def initialize(*args)
+          @args = args
+        end
+        
+        def each(&block)
+          @args.each(&block)
+        end
+        
+        def all_num?
+          @args.all? {|v| v.kind_of? Numeric }
+        end
+        
+        def cast_for_comparison
+          @args.map {|v| floatify(v) }
+        end
+        
+        def bool_cast
+          @args.map {|v| v.to_s.strip == '' ? false : v }
+        end
+        
+        private
+        def floatify(arg)
+          return arg.to_f if arg.kind_of? Numeric
+          return arg.to_s.to_f if arg.to_s =~ /^-?\d+\.?\d*$/ 
+          return arg
+        end        
+      end
+      
+      module Prefix
+        Neg = lambda {|rval| 
+          if rval.kind_of? Numeric 
+            return -rval
+          else
+            return NaN
+          end
+        }
+        
+        Not = lambda {|rval|
+          args = ArgList.new(rval)
+          cr = args.bool_cast[0]
+          return (not(cr))
+        }
+        
+        OpTable = {
+          :- => Neg,
+          :not => Not
+        }
+      end # module Prefix
+
+      # Actual functions to evaluate binary expressions such as
+      # 1+1 and 5%3.
+      # Here is where we get to say what our operators actually do!
+      module Binary
+        Plus = lambda {|lval, rval|
+          args = ArgList.new(lval, rval)
+          return lval+rval if args.all_num?
+          return NaN
+        }
+        
+        Minus = lambda {|lval, rval|
+          args = ArgList.new(lval, rval)
+          return lval-rval if args.all_num?
+          return NaN
+        }
+        
+        Times = lambda {|lval, rval|
+          args = ArgList.new(lval, rval)
+          return lval*rval if args.all_num?
+          return NaN
+        }
+        
+        Div = lambda {|lval, rval|
+          args = ArgList.new(lval, rval)
+          return NaN if rval.to_f == 0.0
+          return lval.to_f/rval.to_f
+        }
+        
+        Mod = lambda {|lval, rval|
+          args = ArgList.new(lval, rval)
+          return lval%rval if args.all_num?
+          return NaN
+        }
+        
+        Concat = lambda {|lval, rval|
+          return lval.to_s+rval.to_s
+        }
+        
+        And = lambda {|lval, rval|
+          args = ArgList.new(lval, rval)
+          cl, cr = args.bool_cast
+          return (cl and cr)
+        }
+        
+        Or = lambda {|lval, rval|
+          args = ArgList.new(lval, rval)
+          cl, cr = args.bool_cast
+          return (cl or cr)
+        }
+        
+        Equals = lambda {|lval, rval|
+          args = ArgList.new(lval, rval)
+          cl, cr = args.cast_for_comparison
+          return cl == cr
+        }
+        
+        NotEquals = lambda {|lval, rval|
+          args = ArgList.new(lval, rval)
+          cl, cr = args.cast_for_comparison
+          return cl != cr
+        }
+        
+        GreaterThan = lambda {|lval, rval|
+          args = ArgList.new(lval, rval)
+          cl, cr = args.cast_for_comparison
+          begin
+            return cl > cr
+          rescue ArgumentError => e
+            return false
+          end
+        }
+        
+        LessThan = lambda {|lval, rval|
+          args = ArgList.new(lval, rval)
+          cl, cr = args.cast_for_comparison
+          begin
+            return cl < cr
+          rescue ArgumentError => e
+            return false
+          end
+        }
+        
+        GreaterThanEquals = lambda {|lval, rval|
+          args = ArgList.new(lval, rval)
+          cl, cr = args.cast_for_comparison
+          begin
+            return cl >= cr
+          rescue ArgumentError => e
+            return false
+          end
+        }
+        
+        LessThanEquals = lambda {|lval, rval|
+          args = ArgList.new(lval, rval)
+          cl, cr = args.cast_for_comparison
+          begin
+            return cl <= cr
+          rescue ArgumentError => e
+            return false
+          end
+        }
+
+        OpTable = {
+          :+ => Plus,
+          :- => Minus,
+          :* => Times,
+          :/ => Div,
+          :% => Mod,
+          :& => Concat,
+          :and => And,
+          :or => Or,
+          '='.to_sym => Equals,
+          '!='.to_sym => NotEquals,
+          :> => GreaterThan,
+          :< => LessThan,
+          :>= => GreaterThanEquals,
+          :<= => LessThanEquals
+        }
+      end
+    end
+  end
+end

data/lib/expression_parser/expressions.rb

@@ -0,0 +1,173 @@
+# Part of the Optimus package for managing E-Prime data
+# 
+# Copyright (C) 2008-09 Board of Regents of the University of Wisconsin System
+# 
+# Written by Nathan Vack <njvack@wisc.edu>, at the Waisman Laborotory for Brain
+# Imaging and Behavior, University of Wisconsin - Madison
+#
+# Expressions for the ParsedCalculator
+require 'expression_parser/evaluators'
+
+module Optimus
+  module ParsedCalculator
+    NaN = 0.0/0.0
+    class Expr
+      # All of our literals, etc will ineherit from Expr. This will imbue
+      # them with the magic to work with our unary and binary operators.
+      BINARY_OPERATORS=[:+, :-, :*, :/, :%, :&, :>, :>=, :<, :<=]
+      BINARY_OPERATORS.each do |op|
+        define_method(op) { |other|
+          return BinaryExpr.new(self, op, other)
+        }
+      end
+      
+      def logical_and(other)
+        return BinaryExpr.new(self, :and, other)
+      end
+      
+      def logical_or(other)
+        return BinaryExpr.new(self, :or, other)
+      end
+      
+      def eq(other)
+        return BinaryExpr.new(self, "=".to_sym, other)
+      end
+      
+      def neq(other)
+        return BinaryExpr.new(self, '!='.to_sym, other)
+      end
+      
+      # Prefixes
+      def -@
+        return PrefixExpr.new(:-, self)
+      end
+      
+      def logical_not
+        return KeywordPrefixExpr.new(:not, self)
+      end
+      
+      def to_bool(*args)
+        val = evaluate(*args)
+        return false if val == ''
+        return val
+      end
+    end
+
+    class BinaryExpr < Expr
+      include Evaluators::Binary
+      
+      attr_reader :left, :op, :right
+      def initialize(left, op, right)
+        @left = left
+        @op = op
+        @right = right
+      end
+  
+      def to_s
+        "(#{@left} #{@op} #{@right})"
+      end
+      
+      def evaluate(*args)
+        lval = @left.evaluate(*args)
+        rval = @right.evaluate(*args)
+        return OpTable[@op].call(lval, rval)
+      end
+    end
+
+    class PrefixExpr < Expr
+      include Evaluators::Prefix
+      attr_reader :op, :right
+      def initialize(op, right)
+        @op = op
+        @right = right
+      end
+  
+      def to_s
+        "#{@op}(#{@right})"
+      end
+      
+      def evaluate(*args)
+        rval = @right.evaluate(*args)
+        return OpTable[@op].call(rval)
+      end
+      
+    end
+    
+    class KeywordPrefixExpr < PrefixExpr
+      def to_s
+        "#{@op} (#{@right})"
+      end
+    end
+
+    class NumberLiteral < Expr
+      def initialize(token)
+        @token = token
+      end
+  
+      def to_s
+        @token
+      end
+      
+      def evaluate(*args)
+        @token.to_f
+      end
+    end
+
+    class StringLiteral < Expr
+      
+      def initialize(token)
+        @token = token
+      end
+  
+      def to_s
+        "'#{@token}'"
+      end
+      
+      def evaluate(*args)
+        @token
+      end
+    end
+
+    class ColumnReference < Expr
+      def initialize(name)
+        @name = name
+      end
+  
+      def to_s
+        "{#{@name}}"
+      end
+      
+      def evaluate(*args)
+        options = args.last || {}
+        row = options[:row] || {}
+        computed_columns = options[:computed_columns] || {}
+        seen_columns = options[:seen_columns] || []
+        if seen_columns.include?(@name)
+          raise EvaluationLoopError.new(
+            "Loop error - #{@name} depends on itself: [#{seen_columns.join(', ')}]"
+          )
+        end
+        
+        if !row[@name] and computed_columns.include?(@name)
+          row[@name] = computed_columns[@name].evaluate(
+            :row => row, 
+            :computed_columns => computed_columns,
+            :seen_columns => ([@name] + seen_columns)
+          )
+        end
+        return magic_cast(row[@name])
+      end
+      
+      private
+      def magic_cast(value)
+        new_val = value.to_s.strip
+        new_val = new_val.to_f if numlike?(new_val)
+        return new_val
+      end
+      
+      def numlike?(value)
+        value.to_s =~ /^-?\d+\.?\d*$/
+      end
+    end # class ColumnReference
+  end
+end

data/lib/log_file_parser.rb

@@ -1,20 +1,20 @@
 # Part of the Optimus package for managing E-Prime data
 # 
-# Copyright (C) 2008 Board of Regents of the University of Wisconsin System
+# Copyright (C) 2008-09 Board of Regents of the University of Wisconsin System
 # 
 # Written by Nathan Vack <njvack@wisc.edu>, at the Waisman Laborotory for Brain
 # Imaging and Behavior, University of Wisconsin - Madison
 
-module Eprime
+module Optimus
   class Reader
     
     # Reads and parses E-Prime log files (the ones that start with 
-    # *** Header Start ***) and transforms them into an Eprime::Data structure
+    # *** Header Start ***) and transforms them into an Optimus::Data structure
     
     
     class LogfileParser
-      # Handles parsing eprime log files, which are essentially a blow-by-blow
-      # log of everything that happened during an eprime run.
+      # Handles parsing optimus log files, which are essentially a blow-by-blow
+      # log of everything that happened during an optimus run.
 
       FRAME_START = '*** LogFrame Start ***'
       FRAME_END = '*** LogFrame End ***'
@@ -46,7 +46,7 @@ module Eprime
         set_counters!
       end
       
-      def to_eprime
+      def to_optimus
         begin
           if @frames.nil? or @frames.empty?
             make_frames!
@@ -56,7 +56,7 @@ module Eprime
         end
     
         @columns ||= @found_cols.names
-        data = Eprime::Data.new(@columns)
+        data = Optimus::Data.new(@columns)
         self.leaf_frames.each do |frame|
           row = data.add_row
           @found_cols.names_with_cols.each do |pair|
@@ -78,7 +78,7 @@ module Eprime
       end
       
       def self.can_parse?(lines)
-        lines[0].include?('*** Header Start ***')
+        lines[0].include?(HEADER_START)
       end
       
       private
@@ -294,4 +294,4 @@ module Eprime
       end
     end # Class LogfileParser
   end # Class Reader
-end # Module Eprime
+end # Module Optimus