optimus-ep 0.6.91 → 0.8.0

data/lib/transformers/column_calculator.rb

@@ -1,381 +1,186 @@
 # Part of the Optimus package for managing E-Prime data
 # 
-# Copyright (C) 2008 Board of Regents of the University of Wisconsin System
+# Copyright (C) 2008-09 Board of Regents of the University of Wisconsin System
 # 
 # Written by Nathan Vack <njvack@wisc.edu>, at the Waisman Laborotory for Brain
 # Imaging and Behavior, University of Wisconsin - Madison
 
-require 'calculator'
-
-module Eprime
+require 'parsed_calculator'
+module Optimus
   module Transformers
 
-    # This implements columnwise and accumulator-style calculations for
-    # Eprime data. It generally allows four main kinds of columns:
+    # This implements columnwise operations with a new shiny parser 
+
+    # Column types:
     # 1: Data columns -- columns backed directly by data
     # 2: Computed columns -- columns computed by numerical operations of other columns in the same row
-    # 3: Copydown columns -- Columns equal to the last non-empty value of another column
-    # 4: Counter columns -- Columns that change value based on the contents of other columns -- generally to count.
-    #
-    # It's worth noting: columns may depend on other columns, as long as the dependency isn't circular.
-    # Currently, counter columns may behave strangely when used in and using computed columns -- a parser
-    # like the computed columns' parser is really needed.
+  
+    # note: to determine if number: ^(((\d{1,3})(,\d{3})*)|(\d+))(.\d+)?$
+    # Better: If you've got funny numbers, coerce into numericness. Make
+    # coercion very robust.
   
     class ColumnCalculator
-      attr_writer :data
-      attr_reader :columns
-    
-      COLUMN_TYPES = %w(data_cols computed_cols copydown_cols counter_cols)
+      attr_accessor :data
+      attr_accessor :sort_expression
+      
       include Enumerable
-    
-      def initialize()
-        @columns = []
-        @columns_intern = []
-        @column_indexes = {}
-        @computed = nil
-        @rows = []
-        COLUMN_TYPES.each do |type|
-          instance_variable_set("@#{type}", [])
-        end
-        # The name 'sorter' is never used; it's just an arbitrary placeholder
-        @sorter = ComputedColumn.new('sorter', Expression.new('1'))
-      end
-    
-      # Makes this into a static Eprime::Data object
-      def to_eprime_data
-        computed
+      
+      DEFAULT_COL_OPTS = {
+        :reset_when => true,
+        :count_when => false,
+        :count_by => :next
+      }
+      def initialize(parser = Optimus::ParsedCalculator::ExpressionParser.new)
+        @computed_column_names = []
+        @computed_columns = {}
+        @computed_data = nil
+        @parser = parser
+        @sort_expression = nil
       end
-    
+      
       def data=(data)
         @data = data
-        @data_cols = []
-        @data.columns.each do |col_name|
-          @data_cols << DataColumn.new(col_name, @data)
-        end
-        set_columns!
+        reset!
       end
-    
-      def [](index)
-        computed[index]
+      
+      def sort_expression=(value)
+        @sort_expression = Evaluatable.new(value, @parser)
       end
-    
-      def column_index(col_id)
-        return @column_indexes[col_id]
+      
+      def computed_column(name, start_val, options = {})
+        if columns.include?(name)
+          raise DuplicateColumnError.new("Can't add duplicate column name #{name}")
+        end
+        sve = Evaluatable.new(start_val, @parser)
+        @computed_column_names << name
+        new_opts = DEFAULT_COL_OPTS.merge(options)
+        DEFAULT_COL_OPTS.keys.each do |key|
+          new_opts[key] = Evaluatable.new(new_opts[key], @parser)
+        end
+        @computed_columns[name] = ComputedColumn.new(
+          name, sve, new_opts
+        )
+        reset!
       end
       
-      alias :find_column_index :column_index
-    
-      def column(col_id)
-        index = column_index(col_id)
-        raise IndexError.new("#{col_id} does not exist") if index.nil?
-        return @columns_intern[index]
+      def copydown_column(new_name, old_name)
+        computed_column(new_name, "{#{old_name}}", :reset_when => "{#{old_name}}")
       end
       
-      def size
-        @data.size
+      def counter_column(name, start_val = 1, options = {})
+        computed_column(name, start_val, options = {})
       end
       
       def columns
-        @columns.dup
-      end
-    
-      def computed_column(name, expression)
-        @computed_cols << ComputedColumn.new(name, Expression.new(expression))
-        set_columns!
-      end
-    
-      def copydown_column(name, copied_name)
-        @copydown_cols << CopydownColumn.new(name, copied_name)
-        set_columns!
-      end
-    
-      def counter_column(name, options = {})
-        @counter_cols << CounterColumn.new(name, options)
-        set_columns!
-      end
-    
-      def sort_expression=(expr)
-        # The name 'sorter' is utterly arbitrary and never used
-        @sorter = ComputedColumn.new('sorter', Expression.new(expr))
-        @computed = nil
-      end
-    
-      def sort_expression
-        @sorter.to_s
-      end
-    
-      def each(&block)
-        computed.each(&block)
-      end
-    
-      def self.compute(numeric_expression)
-        @@calculator.compute(numeric_expression)
+        @data.columns + @computed_column_names
       end
-    
-      private
       
-      def computed
-        @computed || compute_data!
+      def each(&block)
+        computed_data.each(&block)
       end
       
-      def add_column(column)
-        # Raise an error if the column already exists
-        if @column_indexes[column.name]
-          raise ComputationError.new("#{column.name} already exists!")
-        end
-        # Save the index
-        @column_indexes[@columns_intern.size] = @columns_intern.size
-        @column_indexes[column.name] = @columns_intern.size
-        @columns_intern << column
-        @columns << column.name
+      def [](index)
+        computed_data[index]
       end
       
-      def set_columns!
-        @columns = []
-        @columns_intern = []
-        @column_indexes = {}
-        COLUMN_TYPES.each do |type|
-          ar = instance_variable_get("@#{type}")
-          ar.each do |col|
-            add_column(col)
-          end
-        end
-        @computed = nil
-      end
+      private
       
-      # Creates the infix calculator -- called at class instantiation time
-      def self.make_calculator
-        @@calculator = ::Eprime::Calculator.new
-      end
-      # And the class is instantiated NOW!
-      make_calculator
-    
-      # Run through and compute all data in the set. We need to go in order,
-      # because copydown and counter columns depend on the values of previous
-      # rows.
-      def compute_data!
-        @computed = Eprime::Data.new(columns)
-        @data.each_index do |i|
-          row = Row.new(self, @data[i])
-          # Loop over each column type -- it's still (slighyly) important that
-          # we go over each column type specifically. When counter columns
-          # work better, we can rearchitect this a bit.
-          COLUMN_TYPES.each do |type|
-            ar = instance_variable_get("@#{type}")
-            ar.each do |col|
-              row.compute(col.name)
-            end
+      def reset!
+        @computed_data = nil
+      end
+      
+      # Strategy: Compute everything and return it. No lazy-evaluation stuff
+      # to worry about -- we just return a vanilla Optimus::Data object.
+      def computed_data
+        return @computed_data if @computed_data
+        @computed_data = Optimus::Data.new(columns)
+        @computed_data.merge!(@data)
+        @computed_data.each do |row|
+          @computed_column_names.each do |col|
+            row[col] = @computed_columns[col].evaluate(
+              :row => row, :computed_columns => @computed_columns
+            )
           end
-          # Set the sort column -- run it as compute_without_check;
-          # compute() would check the row's ordinary values for a 
-          # 'sorter' column and fail.
-          sv = @sorter.compute_without_check(row)
-          # make it a float, so we don't need to
-          # sort on strings like "-31" (which doesn't work)
-          begin
-            sv = Kernel.Float(sv)
-          rescue ArgumentError
-            # If this fails, it's OK -- we just won't convert.
+          if @sort_expression.respond_to? :evaluate
+            row.sort_value = @sort_expression.evaluate(
+              :row => row, :computed_columns => @computed_columns
+            )
           end
-          new_row = @computed.add_row
-          new_row.sort_value = sv
-          new_row.values = row.values
         end
-        @computed
       end
       
-      class Column
-        attr_accessor :name
-      
-        def initialize(name)
+      class ComputedColumn
+        COUNTERS = {
+          :next => lambda {|val|
+            return val.succ if val.respond_to? :succ
+            return val + 1
+          }
+        }
+        
+        def initialize(name, reset_exp, options = {})
           @name = name
+          @reset_exp = reset_exp
+          @reset_when = options[:reset_when]
+          @count_when = options[:count_when]
+          @count_by = options[:count_by]
+          @current_value = nil
         end
-      
-        # This should be overridden by subclasses
-        def compute(row, path = [])
-          return row[@name]
-        end
-      end
-    
-      class DataColumn < Column
-        def initialize(name, data)
-          @data_index = data.find_column_index(name)
-          @data = data
-          super(name)
-        end
-      end
-    
-      class CopydownColumn < Column
-        def initialize(name, copied_name)
-          super(name)
-          @last_val = ''
-          @copied_name = copied_name
-        end
-      
-        def compute(row, path = [])
-          if !row[@copied_name].to_s.empty?
-            @last_val = row[@copied_name].to_s
+        
+        def evaluate(*args)
+          if @reset_when.bool_eval(*args)
+            @current_value = @reset_exp.evaluate(*args)
           end
-          return @last_val
-        end
-      end
-    
-      class ComputedColumn < Column
-      
-        def initialize(name, expression)
-          @expression = expression
-          super(name)
+          next_val! if @count_when.bool_eval(*args)
+          return @current_value
         end
-      
-        def compute(row, path = [])
-          return super(row) if super(row)
         
-          return compute_without_check(row, path)
-        end
-      
-        def compute_without_check(row, path = [])
-          compute_str = @expression.to_s
-          if path.include?(@name) 
-            raise ComputationError.new("#{compute_str} contains a loop with #{@name} -- can't compute")
-          end
-          
-          # Allow defining the column computation as a lambda
-          return @expression.expr.call(row) if @expression.expr.respond_to? :call
-            
-
-          column_names = @expression.columns
-          column_names.each do |col_name|
-            col = row.find_column(col_name)
-            val = col.compute(row, path+[@name])
-            if val.to_s.empty?
-              val = "0"
-            end
-            compute_str.gsub!("{#{col_name}}", val)
+        private
+        def next_val!(*args)
+          return if @current_value.nil?
+          cval = @count_by.evaluate(*args)
+          counter = COUNTERS[cval] || cval
+          if counter.respond_to?(:call)
+            @current_value = counter.call(@current_value)
+          else
+              @current_value += counter
           end
-          return ColumnCalculator.compute(compute_str)
         end
+      end # class ComputedColumn
       
-      end
-    
-      class CounterColumn < Column
-        STANDARD_OPTS = {
-          :start_value  => 0, 
-          :count_by     => :succ, 
-          :count_when   => lambda {|row| true},
-          :reset_when   => lambda {|row| false}
-        }
-        def initialize(name, options)
-          @options = STANDARD_OPTS.merge(options)
-          @start_value = @options[:start_value]
-          @count_by = @options[:count_by]
-          @count_when = @options[:count_when]
-          @reset_when = @options[:reset_when]
-          @current_value = @start_value
-          super(name)
-        end
-      
-        def compute(row, path = [])
-          if @reset_when.call(row)
-            @current_value = @start_value
-          end
-          if @current_value.respond_to? :call
-            @current_value = @current_value.call(row)
+      class Evaluatable
+        def initialize(target, parser = nil)
+          if target.is_a? String
+            @target = parser.parse(target)
+          else
+            @target = target
           end
+        end
         
-          if @count_when.call(row)
-            if @count_by.respond_to? :call
-              @current_value = @count_by.call(@current_value)
-            elsif @count_by.is_a?(Symbol) || @count_by.is_a?(String)
-              @current_value = @current_value.send(@count_by)
-            else
-              @current_value = @current_value + @count_by
-            end
+        def evaluate(*args)
+          # Check for magicality
+          if @target == :once
+            @target = false
+            return true
           end
-        
-          return @current_value
-        end
-      end
-    
-      class Row
-        attr_reader :values
-        attr_accessor :sort_value
-      
-        def initialize(parent, rowdata)
-          @parent = parent
-          @rowdata = rowdata
-          @values = []
-          # Add all the data columns to @values
-          rowdata.columns.each do |dcol_name|
-            index = @parent.column_index(dcol_name)
-            rd = rowdata[dcol_name]
-            begin
-              ci = values[index]
-            rescue Exception => e
-              raise e
-            end
-            @values[index] = rowdata[dcol_name]
+          # If we don't have a call method or an evaluate method, just
+          # this as our own default method.
+          if not [:call, :evaluate].any? {|msg| @target.respond_to? msg}
+            return @target
           end
-          @sort_value = 1
-        end
-      
-        def [](col_id)
-          if @parent.column_index(col_id).nil?
-            raise IndexError.new("#{col_id} does not exist")
+          if @target.respond_to? :call
+            row = (args.last || {})[:row] || []
+            return @target.call(row)
+          else
+            return @target.evaluate(*args)
           end
-          return @values[@parent.column_index(col_id)]
-        end
-      
-        def find_column(column_name)
-          @parent.column(column_name)
         end
-      
-        def columns
-          @parent.columns
-        end
-      
-      
-        # Recursively compute this column name and every column on which it depends
-        def compute(col_name)
-          raise ArgumentError.new("compute requires a column name") unless col_name.is_a? String
         
-          index = @parent.column_index(col_name)
-          col = @parent.column(col_name)
-          val = col.compute(self)
-          @values[index] = val
+        def bool_eval(*args)
+          val = self.evaluate(*args)
+          return false if val == ''
           return val
         end
-      
-        def <=>(other_row)
-          @sort_value <=> other_row.sort_value
-        end
-      end
-    
-    
-      class Expression
-        attr_reader :columns
-        attr_reader :expr
-      
-        COLUMN_FINDER = /\{([^}]*)\}/ # Finds strings like {foo} and {bar}
-        def initialize(expr_string)
-          @expr = expr_string
-          unless expr_string.is_a? Proc
-            @columns = find_columns(expr_string).freeze 
-          end
-        end
-      
-        def to_s
-          @expr.to_s.dup
-        end
-      
-        private
-        def find_columns(str)
-          return str.scan(COLUMN_FINDER).flatten
-        end
-      end
-    
-      class ComputationError < Exception
-      
-      end
-    end
+      end# class Evaluatable
+    end # class ColumnCalculator
   end
-end
+end

data/lib/transformers/multipasser.rb

@@ -1,6 +1,6 @@
 # Part of the Optimus package for managing E-Prime data
 # 
-# Copyright (C) 2008 Board of Regents of the University of Wisconsin System
+# Copyright (C) 2008-09 Board of Regents of the University of Wisconsin System
 # 
 # Written by Nathan Vack <njvack@wisc.edu>, at the Waisman Laborotory for Brain
 # Imaging and Behavior, University of Wisconsin - Madison
@@ -20,14 +20,18 @@
 
 require 'transformers/column_calculator'
 require 'transformers/row_filter'
+require 'parsed_calculator'
 
-module Eprime
+module Optimus
   module Transformers
     class Multipasser
 
       class Pass
         attr_accessor :sort_expression, :row_filter, :computed_columns
-        def initialize(sort_expression = "1", row_filter = lambda{|r| true}, computed_columns = [])
+        def initialize(
+            sort_expression = "1", 
+            row_filter = lambda{|r| true}, 
+            computed_columns = [])
           @sort_expression = sort_expression
           @row_filter = row_filter
           @computed_columns = computed_columns
@@ -56,7 +60,7 @@ module Eprime
       end
     
       def add_pass(*args)
-        @computed = nil
+        reset!
         if args[0].instance_of? Pass
           p = args[0]
         else
@@ -80,13 +84,13 @@ module Eprime
           @computed = @data
           return @computed
         end
-        @computed = Eprime::Data.new
+        @computed = Optimus::Data.new
         # Just add a simple pass if we don't have any...
         @passes.each do |pass|
           # We want to duplicate the data, add a sort expression to it, add
           # computed columns, filter it, and then merge it into the complete
           # dataset.
-          #cur_data = @data.to_eprime_data 
+          #cur_data = @data.to_optimus_data 
           comp_data = ColumnCalculator.new
           comp_data.data = @data
           comp_data.sort_expression = pass.sort_expression

data/lib/transformers/row_filter.rb

@@ -1,14 +1,14 @@
 # Part of the Optimus package for managing E-Prime data
 # 
-# Copyright (C) 2008 Board of Regents of the University of Wisconsin System
+# Copyright (C) 2008-09 Board of Regents of the University of Wisconsin System
 # 
 # Written by Nathan Vack <njvack@wisc.edu>, at the Waisman Laborotory for Brain
 # Imaging and Behavior, University of Wisconsin - Madison
 
-module Eprime
+module Optimus
   module Transformers
   
-    # Implements a row-wise filter for eprime data.
+    # Implements a row-wise filter for optimus data.
     # Right now it requires a proc; I'll do something better with a little
     # DSL later.
     class RowFilter
@@ -20,7 +20,7 @@ module Eprime
         @computed = nil
       end
     
-      def to_eprime_data
+      def to_optimus_data
         computed
       end
       
@@ -32,7 +32,7 @@ module Eprime
       
       def computed
         return @computed if @computed
-        @computed = Eprime::Data.new(@data.columns)
+        @computed = Optimus::Data.new(@data.columns)
         @data.find_all{ |row|
           match?(row)
         }.each { |row|
@@ -46,13 +46,8 @@ module Eprime
       def match?(row)
         if @filter.is_a? Proc
           return @filter.call(row)
-        elsif @filter.is_a? Array
-          # @filter will be of the form [col_name, comparator, [value]]
-          # only 'equals' is supported for comparators
-          if @filter[1].downcase != 'equals'
-            raise ArgumentError.new('Only equals is supported in filtering')
-          end
-          return row[@filter[0]].to_s == @filter[2].to_s
+        elsif @filter.respond_to? :to_bool
+          return @filter.to_bool(:row => row)
         end
       end
     end

data/lib/transformers/timing_extractor.rb

@@ -1,6 +1,6 @@
 # Part of the Optimus package for managing E-Prime data
 # 
-# Copyright (C) 2008 Board of Regents of the University of Wisconsin System
+# Copyright (C) 2008-09 Board of Regents of the University of Wisconsin System
 # 
 # Written by Nathan Vack <njvack@wisc.edu>, at the Waisman Laborotory for Brain
 # Imaging and Behavior, University of Wisconsin - Madison
@@ -18,7 +18,7 @@
 # the guts of the logic to extract stimuli.
 require 'transformers/basic_transformer'
 
-module Eprime
+module Optimus
   module Transformers
     class TimingExtractor < BasicTransformer
       def initialize(data)
@@ -54,7 +54,7 @@ module Eprime
       
       def extract!
         return if @extracted_data
-        @extracted_data = Eprime::Data.new
+        @extracted_data = Optimus::Data.new
         @stim_schemas.each do |ss|
           matches = processed.find_all(&ss['row_filter'])
           matches.each do |row|

data/lib/version.rb

@@ -1,8 +1,8 @@
-module Eprime
+module Optimus
   module VERSION
     MAJOR = 0
-    MINOR = 6
-    TINY  = 91
+    MINOR = 8
+    TINY  = 0
     
     STRING = [MAJOR, MINOR, TINY].join('.')
   end