optimus-ep 0.6.91 → 0.8.0

data/lib/optimus.rb

@@ -0,0 +1,30 @@
+# Part of the Optimus package for managing E-Prime data
+# 
+# Copyright (C) 2008-09 Board of Regents of the University of Wisconsin System
+# 
+# Written by Nathan Vack <njvack@wisc.edu>, at the Waisman Laborotory for Brain
+# Imaging and Behavior, University of Wisconsin - Madison
+
+# Add our lib to the search path
+$: << File.expand_path(File.join(File.dirname(__FILE__), "..", "lib"))
+
+require 'optimus_data'
+require 'version'
+require 'tabfile_writer'
+require 'optimus_reader'
+
+module Optimus
+
+  class Error < RuntimeError; end
+
+  # Raised whenever an input file's type can't be detemined by Optimus::Reader
+  class UnknownTypeError < Error; end
+  
+  # Raised whenever an input file seems to be damaged
+  class DamagedFileError < Error; end
+  
+  # Raised when a parse fails due to loops
+  class EvaluationLoopError < Error; end
+  
+  class DuplicateColumnError < Error; end
+end

data/lib/{eprime_data.rb → optimus_data.rb}

@@ -1,27 +1,16 @@
 # Part of the Optimus package for managing E-Prime data
 # 
-# Copyright (C) 2008 Board of Regents of the University of Wisconsin System
+# Copyright (C) 2008-09 Board of Regents of the University of Wisconsin System
 # 
 # Written by Nathan Vack <njvack@wisc.edu>, at the Waisman Laborotory for Brain
 # Imaging and Behavior, University of Wisconsin - Madison
 
-module Eprime
+module Optimus
   
-  # Raised when columns were specified at initialization time, and a novel
-  # column is added. Generally, this is an indication that Something is Funny.
-  class ColumnAddedWarning < Exception
-    # We want to be able to get the index out of this
-    attr_reader :index
-    def initialize(message, index)
-      @index = index
-      super(message)
-    end
-  end
-  
-  # A generalized data structure for eprime files -- essentially just
+  # A generalized data structure for optimus files -- essentially just
   # a table structure.
   # I should be able to say:
-  # e_data = Eprime::Data.new
+  # e_data = Optimus::Data.new
   # e_data[0][0] for the first row / col
   # e_data[0]['ExperimentName'] for the experiment name
   # e_data[0][0] = "foo"
@@ -39,7 +28,7 @@ module Eprime
   # One last thing: if you care about column ordering, but may be adding
   # data in an arbitrary order (example: reading E-Prime log files),
   # you can force a column order by passing an array of strings to
-  # Eprime::Data.new  
+  # Optimus::Data.new  
   
   class Data
     
@@ -57,15 +46,15 @@ module Eprime
       end
     end
     
-    # Returns a new Eprime::Data object containing the data from this
+    # Returns a new Optimus::Data object containing the data from this
     # and all other data sets
     def merge(*datasets)
       cols = [self, *datasets].map { |d| d.columns }.flatten.uniq
-      d = Eprime::Data.new(cols)
+      d = Optimus::Data.new(cols)
       return d.merge!(self, *datasets)
     end
     
-    # Combine more Eprime::Data objects into this one, in-place
+    # Combine more Optimus::Data objects into this one, in-place
     def merge!(*datasets)
       datasets.each do |source|
         add_columns!(source.columns)
@@ -99,10 +88,10 @@ module Eprime
     end
     
     def dup
-      Eprime::Data.new().merge!(self)
+      Optimus::Data.new().merge!(self)
     end
     
-    alias :to_eprime_data :dup
+    alias :to_optimus_data :dup
     
     # We mostly delegate to our rows array
     def method_missing(method, *args, &block)
@@ -130,13 +119,7 @@ module Eprime
       @columns << col_id
       @column_hash[col_id] = index
       @column_hash[index] = index
-      if @columns_set_in_initialize and not @options[:ignore_warnings]
-        raise ColumnAddedWarning.new(
-          "Error: Added column #{col_id} after specifying columns at init", 
-          index
-        )
-      end
-      return index
+      index
     end
     
     private
@@ -159,7 +142,7 @@ module Eprime
       def [](index)
         num_index = @parent.find_column_index(index)
         unless (num_index.is_a?(Fixnum) and @parent.columns.length>num_index)
-          raise IndexError.new("Column #{num_index} does not exist")
+          raise IndexError.new("Column #{index} does not exist")
         end
         return @data[num_index]
       end

data/lib/{eprime_reader.rb → optimus_reader.rb}

@@ -1,26 +1,28 @@
 # Part of the Optimus package for managing E-Prime data
 # 
-# Copyright (C) 2008 Board of Regents of the University of Wisconsin System
+# Copyright (C) 2008-09 Board of Regents of the University of Wisconsin System
 # 
 # Written by Nathan Vack <njvack@wisc.edu>, at the Waisman Laborotory for Brain
 # Imaging and Behavior, University of Wisconsin - Madison
 
+require 'stringio'
+require 'iconv'
+
 require 'log_file_parser'
 require 'excel_parser'
 require 'eprimetab_parser'
 require 'raw_tab_parser'
 
-module Eprime
+module Optimus
   
   # A class that should open any type of E-Prime text file and read it into
   # an E-Prime data structure.
-  # This class isn't yet used anywhere.
   class Reader
     
     attr_reader :type, :parser, :input
     attr_accessor :options
     
-    PARSERS = [LogfileParser, ExcelParser, EprimetabParser, RawTabParser]
+    PARSERS = [LogfileParser, ExcelParser, OptimustabParser, RawTabParser]
 
     def initialize(input = nil, options = {})
       @options = options || {}
@@ -31,9 +33,9 @@ module Eprime
       set_input(input)
     end
     
-    def eprime_data
-      @eprime_data ||= @parser.to_eprime
-      return @eprime_data
+    def optimus_data
+      @optimus_data ||= @parser.to_optimus
+      return @optimus_data
     end
     
     def options=(options)
@@ -55,11 +57,23 @@ module Eprime
       end
     end
     
+    def set_file_with_encoding(file)
+      file.rewind
+      datas = file.read(2)
+      if datas == "\377\376"
+        converted = Iconv.conv("ASCII//TRANSLIT", "UTF-16LE", file.read)
+        converted_f = StringIO.new(converted)
+      else
+        converted_f = file
+        converted_f.rewind
+      end
+      return converted_f
+    end
     
-    # Sets @type to one of Eprime::Reader::TYPES or raises an Eprime::UnknownTypeError
-    # Does not change file position.
+    # Sets @type to one of Optimus::Reader::TYPES or raises an 
+    # Optimus::UnknownTypeError. Does not change file position.
     def set_type(file)
-      @file = file
+      @file = set_file_with_encoding(file)
       original_pos = @file.pos
       @file.rewind
       first_lines = Array.new
@@ -73,11 +87,11 @@ module Eprime
       if @type.nil?
         raise UnknownTypeError.new("Can't determine the type of #{file.path}")
       end
-      @eprime_data = nil
+      @optimus_data = nil
       @parser = @type.new(@file, @options)
     end
     
-    # Determines the type of an eprime file, based on its first two lines.
+    # Determines the type of an optimus file, based on its first two lines.
     # Returns one of the elements of PARSERS or nil
     def determine_file_type(first_lines)
       # Log files start with *** Header Start ***
@@ -85,7 +99,7 @@ module Eprime
       # Excel files have a filename on the first line (no tabs); the second line
       # contains at least three elements, tab-delimted
       #
-      # eprime CSV files will have at least three tab-delimited elements on the first line
+      # optimus CSV files will have at least three tab-delimited elements on the first line
       
       return PARSERS.detect { |parser_class| 
         parser_class.can_parse?(first_lines)

data/lib/parsed_calculator.rb

@@ -0,0 +1,92 @@
+# Part of the Optimus package for managing E-Prime data
+# 
+# Copyright (C) 2008-09 Board of Regents of the University of Wisconsin System
+# 
+# Written by Nathan Vack <njvack@wisc.edu>, at the Waisman Laborotory for Brain
+# Imaging and Behavior, University of Wisconsin - Madison
+#
+# A two-stage parser that should be both faster and more flexible than the
+# one-stage regex hack currently in use.
+# Major difference: this will return parse trees that need to be evaluated
+# by passing in a row's worth of data, instead of directly evaluating a string
+# after substituting row data.
+
+require 'rubygems'
+require 'rparsec'
+require 'pp'
+require 'expression_parser/expressions'
+
+module Optimus
+  module ParsedCalculator
+    include RParsec
+    class ExpressionParser
+      extend RParsec::Parsers
+      include RParsec::Parsers
+
+      def initialize
+        @operators = RParsec::Operators.new(
+          %w{+ - * / % & ( ) not and or = != > >= < <=}
+        )
+        expr = nil
+        lazy_expr = lazy { expr }
+        atom = (
+          token(:str) { |lex| StringLiteral.new(lex) } |
+          token(:number) { |lex| NumberLiteral.new(lex) } |
+          token(:column) { |lex| ColumnReference.new(lex) } )
+        
+        lit = atom | (@operators['('] >> lazy_expr << @operators[')'])
+        
+        # Follows hte standard C 
+        table = RParsec::OperatorTable.new.
+          prefix(@operators['-'] >>   lambda {|a| -a}, 50).
+          prefix(@operators['not'] >> lambda {|a| a.logical_not}, 50).
+          infixl(@operators['*'] >>   lambda {|a, b| a*b}, 30).
+          infixl(@operators['/'] >>   lambda {|a, b| a/b}, 30).
+          infixl(@operators['%'] >>   lambda {|a, b| a%b}, 30).
+          infixl(@operators['+'] >>   lambda {|a, b| a+b}, 27).
+          infixl(@operators['-'] >>   lambda {|a, b| a-b}, 27).
+          infixl(@operators['&'] >>   lambda {|a, b| a&b}, 25).
+          infixl(@operators['>'] >>   lambda {|a, b| a>b}, 17).
+          infixl(@operators['>='] >>  lambda {|a, b| a>=b}, 17).
+          infixl(@operators['<'] >>   lambda {|a, b| a<b}, 17).
+          infixl(@operators['<='] >>  lambda {|a, b| a<=b}, 17).
+          infixl(@operators['='] >>   lambda {|a, b| a.eq(b)}, 15).
+          infixl(@operators['!='] >>  lambda {|a, b| a.neq(b)}, 15).
+          infixl(@operators['and'] >> lambda {|a, b| a.logical_and(b)}, 5).
+          infixl(@operators['or'] >>  lambda {|a, b| a.logical_or(b)}, 4)
+        
+        expr = RParsec::Expressions.build(lit, table)
+        
+        lexeme = tokenizer.lexeme(whitespaces) << eof
+        @parser = lexeme.nested(expr << eof)
+      end
+      
+      def parse(str)
+        begin
+          @parser.parse(str)
+        rescue Exception
+          raise
+        end
+      end
+      
+      private
+      
+      def tokenizer
+        return (
+          string_literal.token(:str) |
+          number.token(:number) |
+          column_reference.token(:column) |
+          @operators.lexer
+        )
+      end
+      
+      def string_literal
+        (char("'") >> (not_char("'")|str("''")).many.fragment << char("'"))
+      end
+      
+      def column_reference
+        (char('{') >> (string('\}')|not_char('}')).many_.fragment << char('}'))
+      end
+    end
+  end
+end

data/lib/raw_tab_parser.rb

@@ -1,16 +1,16 @@
 # Part of the Optimus package for managing E-Prime data
 # 
-# Copyright (C) 2008 Board of Regents of the University of Wisconsin System
+# Copyright (C) 2008-09 Board of Regents of the University of Wisconsin System
 # 
 # Written by Nathan Vack <njvack@wisc.edu>, at the Waisman Laborotory for Brain
 # Imaging and Behavior, University of Wisconsin - Madison
 
 # This almost completely delegates to TabfileParser
-# It differs from EprimetabParser only in that it doesn't skip any lines.
+# It differs from OptimustabParser only in that it doesn't skip any lines.
 
 require 'tabfile_parser'
 
-module Eprime
+module Optimus
   class Reader
     class RawTabParser < TabfileParser
       def initialize(file, options = {})

data/lib/runners/generic_runner.rb

@@ -1,6 +1,6 @@
 # Part of the Optimus package for managing E-Prime data
 # 
-# Copyright (C) 2008 Board of Regents of the University of Wisconsin System
+# Copyright (C) 2008-09 Board of Regents of the University of Wisconsin System
 # 
 # Written by Nathan Vack <njvack@wisc.edu>, at the Waisman Laborotory for Brain
 # Imaging and Behavior, University of Wisconsin - Madison
@@ -9,17 +9,17 @@
 # This class should handle argument processing, file I/O, and such.
 
 
-require 'eprime'
-require 'eprime_reader'
+require 'optimus'
+require 'optimus_reader'
 require 'tabfile_writer'
 require 'transformers/timing_extractor'
 require 'optparse'
 require 'ostruct'
 
-module Eprime
+module Optimus
   module Runners
     class GenericRunner
-      include ::Eprime::Transformers
+      include ::Optimus::Transformers
       
       attr_accessor :out, :err
       def initialize(extractor_class, *args)
@@ -43,10 +43,10 @@ module Eprime
       end
       
       def read_data
-        data = Eprime::Data.new()
+        data = Optimus::Data.new()
         @options.input_files.each do |infile|
           File.open(infile) do |f|
-            data.merge!(Eprime::Reader.new(f).eprime_data)
+            data.merge!(Optimus::Reader.new(f).optimus_data)
           end
         end
         @data = data

data/lib/runners/yaml_template/option_parser.rb

@@ -0,0 +1,33 @@
+# Part of the Optimus package for managing E-Prime data
+# 
+# Copyright (C) 2008-09 Board of Regents of the University of Wisconsin System
+# 
+# Written by Nathan Vack <njvack@wisc.edu>, at the Waisman Laborotory for Brain
+# Imaging and Behavior, University of Wisconsin - Madison
+
+# OptionParser for the yaml_template_runner
+
+
+require 'optimus'
+require 'optparse'
+
+module Optimus
+  module Runners
+    module YamlTemplate
+      class OptimusOptionParser
+        attr_reader :errors
+        def initialize
+          @errors = []
+        end
+      end
+      
+      class OptionParserFactory
+        class << self
+          def build
+            
+          end
+        end
+      end
+    end
+  end
+end

data/lib/runners/yaml_template/runner.rb

@@ -0,0 +1,19 @@
+# Part of the Optimus package for managing E-Prime data
+# 
+# Copyright (C) 2008-09 Board of Regents of the University of Wisconsin System
+# 
+# Written by Nathan Vack <njvack@wisc.edu>, at the Waisman Laborotory for Brain
+# Imaging and Behavior, University of Wisconsin - Madison
+
+# This is the main wrapper class -- 
+
+require 'optimus'
+
+module Optimus
+  module Runners
+    module YamlTemplate
+      class Runner
+      end
+    end
+  end
+end

data/lib/tabfile_parser.rb

@@ -1,21 +1,21 @@
 # Part of the Optimus package for managing E-Prime data
 # 
-# Copyright (C) 2008 Board of Regents of the University of Wisconsin System
+# Copyright (C) 2008-09 Board of Regents of the University of Wisconsin System
 # 
 # Written by Nathan Vack <njvack@wisc.edu>, at the Waisman Laborotory for Brain
 # Imaging and Behavior, University of Wisconsin - Madison
 
 
-module Eprime
+module Optimus
   class Reader
     
-    # This class is for reading tab-delimited Eprime files. (Or, really, any tab-delimited file).
+    # This class is for reading tab-delimited Optimus files. (Or, really, any tab-delimited file).
     # The main option of interest is the :skip_lines option, which specifies how many lines
     # to skip before finding column names. For example:
     #
     # TabfileParser.new(stream, :skip_lines => 1)
     #
-    # is what you'd use for skipping the filename line in a standard eprime Excel file.
+    # is what you'd use for skipping the filename line in a standard optimus Excel file.
     #
     # Note: you'll generally be using subclasses of this, and not manually specifying skip_lines.
     
@@ -26,7 +26,7 @@ module Eprime
         @columns = options[:columns]
       end
       
-      def to_eprime
+      def to_optimus
         lines = @file.readlines
         @skip_lines.times do
           lines.shift
@@ -35,7 +35,7 @@ module Eprime
         file_columns = lines.shift.split("\t").map {|elt| elt.strip }
         expected_size = file_columns.size
         columns = file_columns
-        data = Eprime::Data.new(columns)
+        data = Optimus::Data.new(columns)
         current_line = @skip_lines+1
         lines.each do |line|
           current_line += 1

data/lib/tabfile_writer.rb

@@ -1,6 +1,6 @@
 # Part of the Optimus package for managing E-Prime data
 # 
-# Copyright (C) 2008 Board of Regents of the University of Wisconsin System
+# Copyright (C) 2008-09 Board of Regents of the University of Wisconsin System
 # 
 # Written by Nathan Vack <njvack@wisc.edu>, at the Waisman Laborotory for Brain
 # Imaging and Behavior, University of Wisconsin - Madison
@@ -9,9 +9,9 @@
 # way easier (by handling in-band tabs, etc)
 require 'csv'
 
-module Eprime
+module Optimus
   
-  # Writes an Eprime::Data object as a tab-delmited file -- hopefully exactly
+  # Writes an Optimus::Data object as a tab-delmited file -- hopefully exactly
   # like E-DataAid.
   class TabfileWriter
     
@@ -19,17 +19,17 @@ module Eprime
     # Valid things in the options hash:
     # :write_top_line => true, if you want to include the filename
     #   (if it's a file output stream) as the first line output
-    def initialize(eprime_data, outstream, options = {})
+    def initialize(optimus_data, outstream, options = {})
       standard_options = {
         :write_top_line => false,
         :columns => nil,
         :column_labels => true
       }
       good_opts = standard_options.merge(options)
-      @eprime = eprime_data
+      @optimus = optimus_data
       @outstream = outstream
       @write_top_line = good_opts[:write_top_line]
-      @columns = good_opts[:columns] || @eprime.columns
+      @columns = good_opts[:columns] || @optimus.columns
       @column_labels = good_opts[:column_labels]
     end
     
@@ -43,7 +43,7 @@ module Eprime
         if @column_labels
           tsv << @columns
         end
-        @eprime.each do |row|
+        @optimus.each do |row|
           vals = @columns.map { |col_name| row[col_name] }
           tsv << vals
         end

data/lib/transformers/basic_transformer.rb

@@ -1,22 +1,23 @@
 # Part of the Optimus package for managing E-Prime data
 # 
-# Copyright (C) 2008 Board of Regents of the University of Wisconsin System
+# Copyright (C) 2008-09 Board of Regents of the University of Wisconsin System
 # 
 # Written by Nathan Vack <njvack@wisc.edu>, at the Waisman Laborotory for Brain
 # Imaging and Behavior, University of Wisconsin - Madison
 #
-# This is a complete general eprime transforer -- it supports computed columns
+# This is a complete general optimus transforer -- it supports computed columns
 # row filters, and multipass operation. In general, once you've read a file in,
 # this is probably what you want to use.
 # Essentially, this is just a wrapper around the ColumnCalculator,
 # RowFilter, and Multipasser classes
 
-require 'eprime'
+require 'optimus'
 require 'transformers/column_calculator'
 require 'transformers/row_filter'
 require 'transformers/multipasser'
+require 'parsed_calculator'
 
-module Eprime
+module Optimus
   module Transformers
     class BasicTransformer
       def initialize(data)
@@ -34,34 +35,34 @@ module Eprime
       end
       
       def data=(data)
-        reset!
-        @data = Eprime::Data.new.merge(data)
+        reset_all!
+        @data = Optimus::Data.new.merge(data)
       end
       
       def computed_column(*args)
-        reset!
+        reset_all!
         @computed_columns << args
       end
       
       def copydown_column(*args)
-        reset!
+        reset_all!
         @copydown_columns << args
       end
       
       def counter_column(*args)
-        reset!
+        reset_all!
         @counter_columns << args
       end
       
       def row_filter=(filter)
-        reset!
+        reset_filtered!
         @row_filter = filter
       end
       
       alias :row_filter :row_filter=
       
       def add_pass(*args)
-        reset!
+        reset_all!
         p = Multipasser::Pass.new(*args)
         yield p if block_given?
         @passes << p
@@ -75,30 +76,40 @@ module Eprime
         compute! and return @processed
       end
       
-      def reset!
+      def reset_all!
+        @computed = nil
+        @processed = nil
+        return true
+      end
+      
+      def reset_filtered!
         @processed = nil
         return true
       end
       
       def compute!
         return @processed if @processed
-        cc = ColumnCalculator.new
-        cc.data = @data
-        @computed_columns.each do |c|
-          cc.computed_column *c
-        end
-        @copydown_columns.each do |c|
-          cc.copydown_column *c
-        end
-        @counter_columns.each do |c|
-          cc.counter_column *c
+        
+        if @computed.nil?
+          @computed = ColumnCalculator.new
+          @computed.data = @data
+          @computed_columns.each do |c|
+            @computed.computed_column *c
+          end
+          @copydown_columns.each do |c|
+            @computed.copydown_column *c
+          end
+          @counter_columns.each do |c|
+            @computed.counter_column *c
+          end
         end
-        filtered = RowFilter.new(cc, @row_filter)
+        
+        filtered = RowFilter.new(@computed, @row_filter)
         multi = Multipasser.new(filtered)
         @passes.each do |p|
           multi.add_pass(p)
         end
-        @processed = Eprime::Data.new.merge(multi)
+        @processed = Optimus::Data.new.merge(multi)
       end
     end
   end