optimus-ep 0.5.6 → 0.6.0

data/History.txt

@@ -1,3 +1,11 @@
+== 0.6.0 7/14/2008
+* New features:
+  * Added extract_timings, a script to pull stimulus timing data from eprime
+    files.
+  * Added a GenericRunner -- a class that greatly eases end-to-end
+    transformation of eprime data. See extract_timings for an example of
+    its use.
+
 == 0.5.5 7/09/2008
 * New features:
   * Supports making multiple passes through data, to allow extracting multiple

data/Manifest.txt

@@ -5,6 +5,7 @@ Manifest.txt
 README.txt
 Rakefile
 bin/eprime2tabfile
+bin/extract_timings
 lib/calculator.rb
 lib/eprime.rb
 lib/eprime_data.rb
@@ -12,6 +13,7 @@ lib/eprime_reader.rb
 lib/eprimetab_parser.rb
 lib/excel_parser.rb
 lib/log_file_parser.rb
+lib/runners/generic_runner.rb
 lib/tabfile_parser.rb
 lib/tabfile_writer.rb
 lib/transformers/basic_transformer.rb
@@ -27,6 +29,7 @@ spec/eprime_reader_spec.rb
 spec/eprimetab_parser_spec.rb
 spec/excel_parser_spec.rb
 spec/log_file_parser_spec.rb
+spec/runners/generic_runner_spec.rb
 spec/samples/bad_excel_tsv.txt
 spec/samples/corrupt_log_file.txt
 spec/samples/eprime_tsv.txt
@@ -44,4 +47,5 @@ spec/transformers/basic_transformer_spec.rb
 spec/transformers/column_calculator_spec.rb
 spec/transformers/multipasser_spec.rb
 spec/transformers/row_filter_spec.rb
+spec/transformers/timing_extractor_spec.rb
 spec/writers/stimtimes_writer_spec.rb

data/bin/eprime2tabfile

@@ -84,7 +84,7 @@ module Eprime
       opts.on('-o', '--outfile=OUTFILE', String,
         'The name of the file to create. If this',
         'isn\'t specified, print to the standard',
-        "output."
+        "output"
       ) { |val|
         option_hash[:outfile] = val
       }
@@ -92,14 +92,14 @@ module Eprime
   
       opts.on('-c', '--columns=COLUMN_FILE', String, 
         'A tab-separated file containing the columns',
-        "in the order you want your output."
+        "in the order you want your output"
       ) { |val|
         option_hash[:column_file] = val
       }
       opts.separator ""
       
       opts.on('--filter-columns', TrueClass,
-        'Write out only the columns in COLUMN_FILE.',
+        'Write out only the columns in COLUMN_FILE',
         'Requires the use of --columns'
       ) {
         option_hash[:filter_columns] = true
@@ -109,14 +109,14 @@ module Eprime
 
       opts.on('-a', '--add-filename-line', TrueClass,
         'Print the filename as the first line of',
-        "your output, just like E-DataAid."
+        "your output, just like E-DataAid"
       ) { 
         option_hash[:add_filename_line] = true
       }
       opts.separator ""
 
       opts.on('-f', '--force', TrueClass,
-        "Continue processing even there are errors."
+        "Continue processing even there are errors"
       ) {
         option_hash[:force] = true
       }

data/bin/extract_timings

@@ -0,0 +1,26 @@
+#!/usr/bin/env ruby
+
+# Part of the Optimus package for managing E-Prime data
+# 
+# Copyright (C) 2008 Board of Regents of the University of Wisconsin System
+# 
+# Written by Nathan Vack <njvack@wisc.edu>, at the Waisman Laborotory for Brain
+# Imaging and Behavior, University of Wisconsin - Madison
+
+require 'rubygems'
+require 'optparse'
+gem 'optimus-ep'
+require 'runners/generic_runner'
+
+script_name = File.basename(__FILE__)
+
+begin
+  txr = Eprime::Runners::GenericRunner.new(Eprime::Transformers::TimingExtractor, script_name, ARGV)
+  txr.process!
+rescue ArgumentError => e
+  STDERR.puts e.message
+  exit 1
+rescue Exception => e
+  STDERR.puts e.message
+  exit 2
+end

data/lib/eprime_data.rb

@@ -80,6 +80,14 @@ module Eprime
       return self
     end
     
+    def sort!(&block)
+      @rows = @rows.sort(&block)
+    end
+    
+    def sort_by!(&block)
+      @rows = @rows.sort_by(&block)
+    end
+    
     def dup
       Eprime::Data.new().merge!(self)
     end

data/lib/runners/generic_runner.rb

@@ -0,0 +1,157 @@
+# Part of the Optimus package for managing E-Prime data
+# 
+# Copyright (C) 2008 Board of Regents of the University of Wisconsin System
+# 
+# Written by Nathan Vack <njvack@wisc.edu>, at the Waisman Laborotory for Brain
+# Imaging and Behavior, University of Wisconsin - Madison
+
+# A runner to take eprime data files, chew them through a pesudo-templater, 
+# and produce delicious files for importing into other packages. They'll look
+# like:
+#
+# presented     onset     offset
+# stim1         5992      6493
+# stim2         7294      7981
+#
+# This class should handle argument processing, file I/O, and such.
+
+# TODO: Think up a clever way to make this handle arbitrary transformers
+# Probably this is possible by passing the class of the transformer to
+# this runner, and instance_eval()'ing the template file in its presence.
+
+require 'eprime'
+require 'eprime_reader'
+require 'tabfile_writer'
+require 'transformers/timing_extractor'
+require 'optparse'
+require 'ostruct'
+
+module Eprime
+  module Runners
+    class GenericRunner
+      include ::Eprime::Transformers
+      
+      attr_accessor :out, :err
+      def initialize(extractor_class, script_name, *args)
+        @extractor_class = extractor_class
+        @script_name = script_name
+        @out = STDOUT
+        @err = STDERR
+        @args = args
+        @data = nil
+        @timing_extractor = nil
+      end
+      
+      def process!
+        process_arguments(*@args)
+        validate
+        read_data
+        extract_timings
+        write_timings
+      end
+      
+      def read_data
+        data = Eprime::Data.new
+        @options.input_files.each do |infile|
+          File.open(infile) do |f|
+            new_data = Eprime::Reader.new(f).eprime_data
+            data.merge!(new_data)
+          end
+        end
+        @data = data
+      end
+      
+      def extract_timings
+        @timing_extractor = @extractor_class.new(@data)
+        template_code = ''
+        File.open(@options.template_file) { |f| 
+          template_code = f.read 
+        }
+        @timing_extractor.instance_eval(template_code)
+      end
+      
+      def write_timings
+        if @options.outfile
+          @out = File.open(@options.outfile, 'w')
+        end
+        writer = TabfileWriter.new(
+          @timing_extractor.extracted_data, @out, 
+          {:column_labels => @options.column_labels})
+        begin
+          writer.write
+        rescue Errno::EPIPE => e
+          # This is OK
+        ensure
+          if @options.outfile
+            @out.close
+          end
+        end
+      end
+      
+      def validate
+        if @options.help || @args.flatten.size == 0
+          show_help! and raise Exception.new()
+        end
+        if @options.input_files.empty?
+          raise ArgumentError.new("no input files given\n#{usage}")
+        end
+        if !@options.template_file
+          raise ArgumentError.new("no template file given\n#{usage}")
+        end
+        if !File.readable?(@options.template_file)
+          raise ArgumentError.new("can't read #{@options.template_file}\n#{usage}")
+        end
+        return true
+      end
+      
+      def show_help!
+        @err.puts @op.to_s
+      end
+      
+      def usage
+        "#{@op.banner.to_s} \n#{@script_name} --help for help"
+      end
+      
+      private
+      def process_arguments(*args)
+        @options = OpenStruct.new(
+          :help => false,
+          :outfile => nil,
+          :column_labels => true,
+          :template_file => nil,
+          :input_files => []
+        )
+        
+        op = OptionParser.new() do |op| 
+          op.banner = "Usage: extract_timings --template TEMPLATE_FILE [OPTIONS] INPUT_FILES"
+          op.separator ''
+        
+          op.on('-t', '--template=TEMPLATE_FILE', String,
+            'A template containing commands describing',
+            'how to process these files'
+          ) { |t| @options.template_file = t } 
+          
+          op.separator ''
+          op.on('--[no-]column-labels',
+            'Print column lablels in the first row.',
+            'If not specified, do print labels.'
+          ) { |l| @options.column_labels = l }
+
+          op.separator ''
+          op.on('-o', '--outfile=OUTFILE',
+            "The name of the file to save to. If not",
+            "given, print to standard output."
+          ) { |o| @options.outfile = o }
+
+          op.separator ''
+          op.on_tail('-h', '--help',
+            'Print this message.'
+          ) { |h| @options.help = h }
+        end
+        @options.input_files = op.parse(*args) || []
+        @op = op
+      end
+      
+    end
+  end
+end

data/lib/tabfile_writer.rb

@@ -20,10 +20,17 @@ module Eprime
     # :write_top_line => true, if you want to include the filename
     #   (if it's a file output stream) as the first line output
     def initialize(eprime_data, outstream, options = {})
+      standard_options = {
+        :write_top_line => false,
+        :columns => nil,
+        :column_labels => true
+      }
+      good_opts = standard_options.merge(options)
       @eprime = eprime_data
       @outstream = outstream
-      @write_top_line = options[:write_top_line]
-      @columns = options[:columns] || @eprime.columns
+      @write_top_line = good_opts[:write_top_line]
+      @columns = good_opts[:columns] || @eprime.columns
+      @column_labels = good_opts[:column_labels]
     end
     
     # Write to the output stream.
@@ -33,7 +40,9 @@ module Eprime
           name = @outstream.respond_to?(:path) ? File.expand_path(@outstream.path.to_s) : ''
           tsv << [name]
         end
-        tsv << @columns
+        if @column_labels
+          tsv << @columns
+        end
         @eprime.each do |row|
           vals = @columns.map { |col_name| row[col_name] }
           tsv << vals

data/lib/transformers/basic_transformer.rb

@@ -58,6 +58,8 @@ module Eprime
         @row_filter = filter
       end
       
+      alias :row_filter :row_filter=
+      
       def add_pass(*args)
         reset!
         p = Multipasser::Pass.new(*args)

data/lib/transformers/timing_extractor.rb

@@ -16,14 +16,57 @@
 # In an experiment, this will take, as an argument, a template written in ruby
 # that will be eval'd in the context of this instance -- that will contain
 # the guts of the logic to extract stimuli.
+require 'transformers/basic_transformer'
 
 module Eprime
-  class TimingExtractor
-    def initialize(argv)
+  module Transformers
+    class TimingExtractor < BasicTransformer
+      def initialize(data)
+        super(data)
+        @stim_schemas = []
+        @extracted_data = nil
+      end
       
-    end
-    
-    def extract
+      def extract_stimulus(
+        name_column,
+        onset_column,
+        offset_column,
+        row_filter = (lambda { |r| true }) 
+      )
+        @stim_schemas << {
+          'name_column' => name_column,
+          'onset_column' => onset_column,
+          'offset_column' => offset_column,
+          'row_filter' => row_filter
+        }
+        @extracted_data = nil
+      end
+      
+      def extracted_data
+        extract!
+        return @extracted_data
+      end
+      
+      private
+      def extract_reset!
+        @extracted_data = nil
+      end
+      
+      def extract!
+        return if @extracted_data
+        @extracted_data = Eprime::Data.new
+        @stim_schemas.each do |ss|
+          matches = processed.find_all(&ss['row_filter'])
+          matches.each do |row|
+            nr = @extracted_data.add_row
+            nr['presented'] = row[ss['name_column']]
+            nr['onset'] = row[ss['onset_column']]
+            nr['offset'] = row[ss['offset_column']]
+            nr.sort_value = nr['onset'].to_f
+          end
+        end
+        @extracted_data.sort!
+      end
     end
   end
 end

data/lib/version.rb

@@ -1,8 +1,8 @@
 module Eprime
   module VERSION
     MAJOR = 0
-    MINOR = 5
-    TINY  = 6
+    MINOR = 6
+    TINY  = 0
     
     STRING = [MAJOR, MINOR, TINY].join('.')
   end

data/spec/runners/generic_runner_spec.rb

@@ -0,0 +1,32 @@
+# Part of the Optimus package for managing E-Prime data
+# 
+# Copyright (C) 2008 Board of Regents of the University of Wisconsin System
+# 
+# Written by Nathan Vack <njvack@wisc.edu>, at the Waisman Laborotory for Brain
+# Imaging and Behavior, University of Wisconsin - Madison
+
+# Part of the Optimus package for managing E-Prime data
+# 
+# Copyright (C) 2008 Board of Regents of the University of Wisconsin System
+# 
+# Written by Nathan Vack <njvack@wisc.edu>, at the Waisman Laborotory for Brain
+# Imaging and Behavior, University of Wisconsin - Madison
+
+require File.join(File.dirname(__FILE__),'../spec_helper')
+require File.join(File.dirname(__FILE__), '../../lib/eprime')
+require 'runners/generic_runner'
+include EprimeTestHelper
+
+describe Eprime::Runners::GenericRunner do
+  before :each do
+    @txr = Eprime::Runners::GenericRunner.new
+  end
+  
+  it "should start with stdout in @out" do
+    @txr.out.should == STDOUT
+  end
+  
+  it "should start with stderr in @err" do
+    @txr.err.should == STDERR
+  end
+end

data/spec/tabfile_writer_spec.rb

@@ -88,4 +88,17 @@ describe "Eprime::TabfileWriter" do
       lambda { @writer.write }.should raise_error(IndexError)
     end
   end
+  
+  describe "with column_labels set to false" do
+    before :each do
+      @writer = Eprime::TabfileWriter.new(@eprime_data, @out_s, {:column_labels => false})
+      @writer.write
+      @out_s.rewind
+    end
+    
+    it "should not write a line with column labels" do
+      @out_s.readlines.size.should == @eprime_data.size
+    end
+    
+  end
 end

data/spec/transformers/timing_extractor_spec.rb

@@ -0,0 +1,90 @@
+# Part of the Optimus package for managing E-Prime data
+# 
+# Copyright (C) 2008 Board of Regents of the University of Wisconsin System
+# 
+# Written by Nathan Vack <njvack@wisc.edu>, at the Waisman Laborotory for Brain
+# Imaging and Behavior, University of Wisconsin - Madison
+
+require File.join(File.dirname(__FILE__),'../spec_helper')
+require File.join(File.dirname(__FILE__), '../../lib/eprime')
+require 'transformers/timing_extractor'
+include EprimeTestHelper
+include Eprime::Transformers
+
+describe Eprime::Transformers::TimingExtractor do
+  before :each do
+    @data = mock_edata
+    @tx = TimingExtractor.new(@data)
+  end
+  
+  it "should be a BasicTransformer" do
+    @tx.should be_a_kind_of(BasicTransformer)
+  end
+  
+  it "should accept extract_stimulus" do
+    lambda { 
+      @tx.extract_stimulus('stim_time', 'stim_time', 'stim_time') 
+    }.should_not raise_error
+  end
+  
+  it "should have nothing in extracted_data when no stimuli are extracted" do
+    @tx.extracted_data.size.should == 0
+  end
+  
+  it "should return rows when extracting stim_time" do
+    # These results will not be very meaningful
+    @tx.extract_stimulus('stim_time', 'stim_time', 'stim_time') 
+    @tx.extracted_data.size.should == @data.size
+  end
+  
+  it "should accept columns" do
+    lambda { @tx.columns }.should_not raise_error
+  end
+  
+  it "should extract from computed columns" do
+    @tx.computed_column 'foo', 'a'
+    @tx.columns.should include('foo')
+    @tx.extract_stimulus('foo', 'foo', 'foo')
+    @tx.extracted_data.size.should == @data.size
+  end
+  
+  it "should honor row filters in stim extraction" do
+    @tx.extract_stimulus(
+      'stim_time', 
+      'stim_time', 
+      'stim_time',
+      lambda {|row| !row['sparse'].to_s.empty? }
+    )
+    count = @data.find_all { |r| !r['sparse'].to_s.empty? }.size
+    @tx.extracted_data.size.should == count
+  end
+  
+  describe "(extracting two stimuli)" do
+    before :each do
+      @data = mock_edata
+      @tx = TimingExtractor.new(@data)
+      @tx.computed_column('stim_name','stim')
+      @tx.computed_column('fix_name', 'fixation')
+      @tx.computed_column('stim_offset', '{stim_time} + 500 - {run_start}')
+      @tx.computed_column('fix_offset', '{fix_time}+130-{run_start}')
+      @tx.extract_stimulus('stim_name', 'stim_time', 'stim_time')
+      @tx.extract_stimulus('fix_name', 'fix_time', 'fix_offset')
+      @ed = @tx.extracted_data
+    end
+    
+    it "should have columns presented, onset, and offset" do
+      @ed.columns.should == %w(presented onset offset)
+    end
+    
+    it "should have twice as many rows as @data" do
+      @ed.size.should == @data.size*2
+    end
+    
+    it "should be ordered by onset" do
+      ordered = @ed.sort_by { |r| r['onset'].to_f }
+      ordered.each_index do |i|
+        ordered[i]['onset'].to_s.should == @ed[i]['onset'].to_s
+      end
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: optimus-ep
 version: !ruby/object:Gem::Version 
-  version: 0.5.6
+  version: 0.6.0
 platform: ruby
 authors: 
 - Nathan Vack
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2008-07-09 00:00:00 -05:00
+date: 2008-07-14 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -26,6 +26,7 @@ email:
 - njvack@freshforever.net
 executables: 
 - eprime2tabfile
+- extract_timings
 extensions: []
 
 extra_rdoc_files: 
@@ -51,6 +52,7 @@ files:
 - README.txt
 - Rakefile
 - bin/eprime2tabfile
+- bin/extract_timings
 - lib/calculator.rb
 - lib/eprime.rb
 - lib/eprime_data.rb
@@ -58,6 +60,7 @@ files:
 - lib/eprimetab_parser.rb
 - lib/excel_parser.rb
 - lib/log_file_parser.rb
+- lib/runners/generic_runner.rb
 - lib/tabfile_parser.rb
 - lib/tabfile_writer.rb
 - lib/transformers/basic_transformer.rb
@@ -73,6 +76,7 @@ files:
 - spec/eprimetab_parser_spec.rb
 - spec/excel_parser_spec.rb
 - spec/log_file_parser_spec.rb
+- spec/runners/generic_runner_spec.rb
 - spec/samples/bad_excel_tsv.txt
 - spec/samples/corrupt_log_file.txt
 - spec/samples/eprime_tsv.txt
@@ -90,6 +94,7 @@ files:
 - spec/transformers/column_calculator_spec.rb
 - spec/transformers/multipasser_spec.rb
 - spec/transformers/row_filter_spec.rb
+- spec/transformers/timing_extractor_spec.rb
 - spec/writers/stimtimes_writer_spec.rb
 has_rdoc: true
 homepage: http://code.google.com/p/optimus-ep