optimus-ep 0.9.1 → 0.10.0

data/CHANGELOG

@@ -1,3 +1,5 @@
+v.0.10.0. Actually make things like counter columns work again. Sorry, anyone who downloaded 0.9.x.
+
 v0.9.1. Change dependency from rparsec to rparsec-ruby19
 
 v0.9.0. Ruby 1.9 compatibility.

data/Manifest

@@ -3,6 +3,7 @@ GPL.txt
 LICENSE
 Manifest
 README
+README.textile
 Rakefile
 autotest/discover.rb
 bin/eprime2tabfile

data/README.textile

@@ -0,0 +1,74 @@
+h1. Optimus
+
+h5. Turning data into results
+
+Optimus is a set of libraries designed to process behavioral data for use in statistical analysis packages. Think of it as the ultimate scriptable spreadsheet.
+
+Currently, it's geared towards processing files generated by "E-Prime":http://www.pstnet.com/eprime.cfm, but really works with anything that produces spreadsheet-like data.
+
+h2. Installing
+
+We're all set in "RubyGems":http://rubygems.org, so: 
+
+<pre>
+sudo gem install optimus-ep
+</pre>
+
+should get you set up with Optimus and the required "RParsec":http://docs.codehaus.org/display/JPARSEC/rparsec+overview. I've developed and tested with Ruby 1.8.7, but suspect 1.9.x will work, too.
+
+h2. Command-line tools:
+
+Right now, there's one program recommended for general use:
+
+h3. eprime2tabfile
+
+Turns the text file written by E-Prime (or, really, any file we can read) into a tab-delimited file. It prints to stdout, and should be redirected if you want to save the output.
+
+Usage:
+
+<pre>
+Usage: eprime2tabfile [options] INPUT_FILES
+
+    -o, --outfile=OUTFILE            The name of the file to create. If this
+                                     isn't specified, print to the standard
+                                     output.
+
+    -c, --columns=COLUMN_FILE        A tab-separated file containing the columns
+                                     in the order you want your output.
+
+        --filter-columns             Write out only the columns in COLUMN_FILE.
+                                     Requires the use of --columns
+
+    -a, --add-filename-line          Print the filename as the first line of
+                                     your output, just like E-DataAid.
+
+    -f, --force                      Continue processing even there are errors.
+
+    -h, --help                       Print this message.
+</pre>
+
+h2. API Usage
+
+Coming very soon.
+
+h2. License
+
+(The GNU Public License, Version 2)
+
+Optimus: Libraries and utilities for interacting with E-Prime data files
+Copyright (C) 2008-09 Board of Regents of the University of Wisconsin System
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of version 2 of the GNU General Public
+License as published by the Free Software Foundation.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+
+E-Prime is a registered trademark of Psychology Software Tools, Inc.

data/lib/optimus.rb

@@ -23,8 +23,11 @@ module Optimus
   # Raised whenever an input file seems to be damaged
   class DamagedFileError < Error; end
   
+  class ParseError < Error; end
+  
   # Raised when a parse fails due to loops
-  class EvaluationLoopError < Error; end
+  class EvaluationLoopError < ParseError; end
   
   class DuplicateColumnError < Error; end
+  
 end

data/lib/optimus_reader.rb

@@ -6,7 +6,6 @@
 # Imaging and Behavior, University of Wisconsin - Madison
 
 require 'stringio'
-require 'iconv'
 
 require 'log_file_parser'
 require 'excel_parser'
@@ -58,11 +57,21 @@ module Optimus
     end
     
     def set_file_with_encoding(file)
+      converted_f = nil
       file.rewind
       datas = file.read(2)
       if datas == "\377\376"
-        converted = Iconv.conv("ASCII//TRANSLIT", "UTF-16LE", file.read)
-        converted_f = StringIO.new(converted)
+        file_data = file.read
+        if file_data.respond_to? :encode
+          # Ruby 1.9.x -- iconv is deprecated here
+          converted_f = StringIO.new(
+            file_data.encode("UTF-8", "UTF-16LE"))
+        else
+          # Ruby 1.8.x -- no String#encode()
+          require 'iconv'
+          converted_f = StringIO.new(
+            Iconv.conv("UTF-8", "UTF-16LE", file_data))
+        end
       else
         converted_f = file
         converted_f.rewind

data/lib/parsed_calculator.rb

@@ -62,11 +62,10 @@ module Optimus
       end
       
       def parse(str)
-        begin
-          @parser.parse(str)
-        rescue Exception
-          raise
+        if str.strip.empty?
+          return ''
         end
+        @parser.parse(str)
       end
       
       private

data/lib/tabfile_writer.rb

@@ -35,7 +35,12 @@ module Optimus
     
     # Write to the output stream.
     def write
-      tsv = CSV.new(@outstream, {:col_sep => "\t"})
+      tsv = nil
+      begin
+        tsv = CSV::Writer.create(@outstream, col_sep="\t")
+      rescue
+        tsv = CSV.new(@outstream, {:col_sep => "\t"})
+      end
       if @write_top_line
         name = @outstream.respond_to?(:path) ? File.expand_path(@outstream.path.to_s) : ''
         tsv << [name]

data/lib/transformers/column_calculator.rb

@@ -47,7 +47,7 @@ module Optimus
         @sort_expression = Evaluatable.new(value, @parser)
       end
       
-      def computed_column(name, start_val, options = {})
+      def computed_column(name, start_val, options={})
         if columns.include?(name)
           raise DuplicateColumnError.new("Can't add duplicate column name #{name}")
         end
@@ -55,11 +55,13 @@ module Optimus
         @computed_column_names << name
         new_opts = DEFAULT_COL_OPTS.merge(options)
         DEFAULT_COL_OPTS.keys.each do |key|
-          new_opts[key] = Evaluatable.new(new_opts[key], @parser)
+          begin
+            new_opts[key] = Evaluatable.new(new_opts[key], @parser)
+          rescue Exception => e
+            raise ParseError.new("Did not understand #{name}: #{e}")
+          end
         end
-        @computed_columns[name] = ComputedColumn.new(
-          name, sve, new_opts
-        )
+        @computed_columns[name] = ComputedColumn.new(name, sve, new_opts)
         reset!
       end
       
@@ -67,8 +69,11 @@ module Optimus
         computed_column(new_name, "{#{old_name}}", :reset_when => "{#{old_name}}")
       end
       
-      def counter_column(name, start_val = 1, options = {})
-        computed_column(name, start_val, options = {})
+      def counter_column(name, options = {})
+        start_val = options[:start_val] || 1
+        options[:reset_when] ||= :once
+        options[:count_when] ||= true
+        computed_column(name, start_val, options)
       end
       
       def columns
@@ -127,11 +132,13 @@ module Optimus
         end
         
         def evaluate(*args)
-          if @reset_when.bool_eval(*args)
+          should_reset = @reset_when.bool_eval(*args)
+          if should_reset
             @current_value = @reset_exp.evaluate(*args)
           end
-          next_val! if @count_when.bool_eval(*args)
-          return @current_value
+          should_count = @count_when.bool_eval(*args)
+          next_val! if should_count
+          @current_value
         end
         
         private
@@ -156,6 +163,10 @@ module Optimus
           end
         end
         
+        def to_s
+          "Evaluatable (#{@target})"
+        end
+        
         def evaluate(*args)
           # Check for magicality
           if @target == :once

data/lib/transformers/row_filter.rb

@@ -9,8 +9,6 @@ module Optimus
   module Transformers
   
     # Implements a row-wise filter for optimus data.
-    # Right now it requires a proc; I'll do something better with a little
-    # DSL later.
     class RowFilter
       include Enumerable
     
@@ -24,6 +22,10 @@ module Optimus
         computed
       end
       
+      def each
+        computed.each {|row| yield row }
+      end
+      
       def method_missing(method, *args, &block)
         computed.send method, *args, &block
       end

data/lib/version.rb

@@ -1,8 +1,8 @@
 module Optimus
   module VERSION
     MAJOR = 0
-    MINOR = 9
-    TINY  = 1
+    MINOR = 10
+    TINY  = 0
     
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

data/optimus-ep.gemspec

@@ -2,17 +2,17 @@
 
 Gem::Specification.new do |s|
   s.name = "optimus-ep"
-  s.version = "0.9.1"
+  s.version = "0.10.0"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 1.2") if s.respond_to? :required_rubygems_version=
   s.authors = ["Nate Vack"]
   s.cert_chain = ["/Users/njvack/.gem/cert/gem-public_cert.pem"]
-  s.date = "2012-02-09"
+  s.date = "2012-02-17"
   s.description = "Utilities to process behavioral data generated by E-Prime"
   s.email = "njvack@wisc.edu"
   s.executables = ["eprime2tabfile", "extract_timings"]
-  s.extra_rdoc_files = ["CHANGELOG", "LICENSE", "README", "bin/eprime2tabfile", "bin/extract_timings", "lib/eprimetab_parser.rb", "lib/excel_parser.rb", "lib/expression_parser/evaluators.rb", "lib/expression_parser/expressions.rb", "lib/log_file_parser.rb", "lib/optimus.rb", "lib/optimus_data.rb", "lib/optimus_reader.rb", "lib/parsed_calculator.rb", "lib/raw_tab_parser.rb", "lib/runners/generic_runner.rb", "lib/runners/yaml_template/option_parser.rb", "lib/runners/yaml_template/runner.rb", "lib/tabfile_parser.rb", "lib/tabfile_writer.rb", "lib/transformers/basic_transformer.rb", "lib/transformers/column_calculator.rb", "lib/transformers/multipasser.rb", "lib/transformers/row_filter.rb", "lib/transformers/timing_extractor.rb", "lib/version.rb", "lib/writers/stimtimes_writer.rb"]
-  s.files = ["CHANGELOG", "GPL.txt", "LICENSE", "Manifest", "README", "Rakefile", "autotest/discover.rb", "bin/eprime2tabfile", "bin/extract_timings", "lib/eprimetab_parser.rb", "lib/excel_parser.rb", "lib/expression_parser/evaluators.rb", "lib/expression_parser/expressions.rb", "lib/log_file_parser.rb", "lib/optimus.rb", "lib/optimus_data.rb", "lib/optimus_reader.rb", "lib/parsed_calculator.rb", "lib/raw_tab_parser.rb", "lib/runners/generic_runner.rb", "lib/runners/yaml_template/option_parser.rb", "lib/runners/yaml_template/runner.rb", "lib/tabfile_parser.rb", "lib/tabfile_writer.rb", "lib/transformers/basic_transformer.rb", "lib/transformers/column_calculator.rb", "lib/transformers/multipasser.rb", "lib/transformers/row_filter.rb", "lib/transformers/timing_extractor.rb", "lib/version.rb", "lib/writers/stimtimes_writer.rb", "spec/eprimetab_parser_spec.rb", "spec/excel_parser_spec.rb", "spec/expression_parser/evaluators_spec.rb", "spec/expression_parser/expressions_spec.rb", "spec/log_file_parser_spec.rb", "spec/optimus_data_spec.rb", "spec/optimus_reader_spec.rb", "spec/parsed_calculator_spec.rb", "spec/raw_tab_parser_spec.rb", "spec/runners/generic_runner_spec.rb", "spec/runners/yaml_template/option_parser_spec.rb", "spec/runners/yaml_template/runner_spec.rb", "spec/samples/bad_excel_tsv.txt", "spec/samples/corrupt_log_file.txt", "spec/samples/eprime_tsv.txt", "spec/samples/excel_tsv.txt", "spec/samples/optimus_log.txt", "spec/samples/optimus_log_utf16le.txt", "spec/samples/raw_tsv.txt", "spec/samples/short_columns.txt", "spec/samples/sorted_columns.txt", "spec/samples/std_columns.txt", "spec/samples/unknown_type.txt", "spec/samples/unreadable_file", "spec/spec_helper.rb", "spec/tabfile_parser_spec.rb", "spec/tabfile_writer_spec.rb", "spec/transformers/basic_transformer_spec.rb", "spec/transformers/column_calculator_spec.rb", "spec/transformers/multipasser_spec.rb", "spec/transformers/row_filter_spec.rb", "spec/transformers/timing_extractor_spec.rb", "spec/writers/stimtimes_writer_spec.rb", "optimus-ep.gemspec"]
+  s.extra_rdoc_files = ["CHANGELOG", "LICENSE", "README", "README.textile", "bin/eprime2tabfile", "bin/extract_timings", "lib/eprimetab_parser.rb", "lib/excel_parser.rb", "lib/expression_parser/evaluators.rb", "lib/expression_parser/expressions.rb", "lib/log_file_parser.rb", "lib/optimus.rb", "lib/optimus_data.rb", "lib/optimus_reader.rb", "lib/parsed_calculator.rb", "lib/raw_tab_parser.rb", "lib/runners/generic_runner.rb", "lib/runners/yaml_template/option_parser.rb", "lib/runners/yaml_template/runner.rb", "lib/tabfile_parser.rb", "lib/tabfile_writer.rb", "lib/transformers/basic_transformer.rb", "lib/transformers/column_calculator.rb", "lib/transformers/multipasser.rb", "lib/transformers/row_filter.rb", "lib/transformers/timing_extractor.rb", "lib/version.rb", "lib/writers/stimtimes_writer.rb"]
+  s.files = ["CHANGELOG", "GPL.txt", "LICENSE", "Manifest", "README", "README.textile", "Rakefile", "autotest/discover.rb", "bin/eprime2tabfile", "bin/extract_timings", "lib/eprimetab_parser.rb", "lib/excel_parser.rb", "lib/expression_parser/evaluators.rb", "lib/expression_parser/expressions.rb", "lib/log_file_parser.rb", "lib/optimus.rb", "lib/optimus_data.rb", "lib/optimus_reader.rb", "lib/parsed_calculator.rb", "lib/raw_tab_parser.rb", "lib/runners/generic_runner.rb", "lib/runners/yaml_template/option_parser.rb", "lib/runners/yaml_template/runner.rb", "lib/tabfile_parser.rb", "lib/tabfile_writer.rb", "lib/transformers/basic_transformer.rb", "lib/transformers/column_calculator.rb", "lib/transformers/multipasser.rb", "lib/transformers/row_filter.rb", "lib/transformers/timing_extractor.rb", "lib/version.rb", "lib/writers/stimtimes_writer.rb", "spec/eprimetab_parser_spec.rb", "spec/excel_parser_spec.rb", "spec/expression_parser/evaluators_spec.rb", "spec/expression_parser/expressions_spec.rb", "spec/log_file_parser_spec.rb", "spec/optimus_data_spec.rb", "spec/optimus_reader_spec.rb", "spec/parsed_calculator_spec.rb", "spec/raw_tab_parser_spec.rb", "spec/runners/generic_runner_spec.rb", "spec/runners/yaml_template/option_parser_spec.rb", "spec/runners/yaml_template/runner_spec.rb", "spec/samples/bad_excel_tsv.txt", "spec/samples/corrupt_log_file.txt", "spec/samples/eprime_tsv.txt", "spec/samples/excel_tsv.txt", "spec/samples/optimus_log.txt", "spec/samples/optimus_log_utf16le.txt", "spec/samples/raw_tsv.txt", "spec/samples/short_columns.txt", "spec/samples/sorted_columns.txt", "spec/samples/std_columns.txt", "spec/samples/unknown_type.txt", "spec/samples/unreadable_file", "spec/spec_helper.rb", "spec/tabfile_parser_spec.rb", "spec/tabfile_writer_spec.rb", "spec/transformers/basic_transformer_spec.rb", "spec/transformers/column_calculator_spec.rb", "spec/transformers/multipasser_spec.rb", "spec/transformers/row_filter_spec.rb", "spec/transformers/timing_extractor_spec.rb", "spec/writers/stimtimes_writer_spec.rb", "optimus-ep.gemspec"]
   s.homepage = "http://github.com/njvack/optimus-ep"
   s.rdoc_options = ["--line-numbers", "--inline-source", "--title", "Optimus-ep", "--main", "README"]
   s.require_paths = ["lib"]

data/spec/parsed_calculator_spec.rb

@@ -16,6 +16,10 @@ describe Optimus::ParsedCalculator::ExpressionParser do
     @exp = Optimus::ParsedCalculator::ExpressionParser.new
   end
   
+  it "should allow empty expressions" do
+    @exp.should round_trip("")
+  end
+  
   it "should parse positive integers" do
     @exp.should round_trip("1")
   end

data/spec/transformers/column_calculator_spec.rb

@@ -40,6 +40,11 @@ describe Optimus::Transformers::ColumnCalculator do
     @pc.columns.should include(NEW_COLUMN)
   end
   
+  it "should allow empty columns" do
+    @pc.computed_column NEW_COLUMN, ""
+    @pc.columns.should include(NEW_COLUMN)
+  end
+  
   it "should raise an error when adding bad columns" do
     lambda {
       @pc.computed_column NEW_COLUMN, "FIAL"
@@ -59,7 +64,9 @@ describe Optimus::Transformers::ColumnCalculator do
   end
   
   it "should return values for columns based on lambdas" do
-    @pc.computed_column NEW_COLUMN, lambda {|row| row['stim_time'] }
+    @pc.computed_column NEW_COLUMN, lambda {|row| 
+      row['stim_time'] 
+    }
     @pc.computed_column "NEW_COL2", lambda {|row| 1}
     @pc.each do |row|
       row[NEW_COLUMN].should == row['stim_time']
@@ -131,7 +138,7 @@ describe Optimus::Transformers::ColumnCalculator do
     it "should never update when false condition" do
       @pc.computed_column NEW_COLUMN, "{stim_time}", :reset_when => "''"
       @pc.each do |row|
-        row[NEW_COLUMN].should be_nil
+        row[NEW_COLUMN].to_s.should == ''
       end
     end
     

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: optimus-ep
 version: !ruby/object:Gem::Version
-  version: 0.9.1
+  version: 0.10.0
   prerelease: 
 platform: ruby
 authors:
@@ -50,11 +50,11 @@ cert_chain:
   -----END CERTIFICATE-----
 
 '
-date: 2012-02-09 00:00:00.000000000 Z
+date: 2012-02-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rparsec-ruby19
-  requirement: &2152782200 !ruby/object:Gem::Requirement
+  requirement: &2157389960 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -62,10 +62,10 @@ dependencies:
         version: '1.0'
   type: :runtime
   prerelease: false
-  version_requirements: *2152782200
+  version_requirements: *2157389960
 - !ruby/object:Gem::Dependency
   name: rspec
-  requirement: &2152781580 !ruby/object:Gem::Requirement
+  requirement: &2157389400 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ! '>='
@@ -73,7 +73,7 @@ dependencies:
         version: '0'
   type: :development
   prerelease: false
-  version_requirements: *2152781580
+  version_requirements: *2157389400
 description: Utilities to process behavioral data generated by E-Prime
 email: njvack@wisc.edu
 executables:
@@ -84,6 +84,7 @@ extra_rdoc_files:
 - CHANGELOG
 - LICENSE
 - README
+- README.textile
 - bin/eprime2tabfile
 - bin/extract_timings
 - lib/eprimetab_parser.rb
@@ -114,6 +115,7 @@ files:
 - LICENSE
 - Manifest
 - README
+- README.textile
 - Rakefile
 - autotest/discover.rb
 - bin/eprime2tabfile