rpipe 0.0.1

data/lib/logfile.rb

@@ -0,0 +1,310 @@
+require 'pp'
+require 'ruport'
+require 'default_logger'
+require 'matlab_helpers/matlab_queue'
+
+###############################################	 START OF CLASS	 ######################################################
+# An object that helps parse and format Behavioral Logfiles.
+class Logfile
+  include DefaultLogger
+  
+  # An array of rows raw text data 
+  attr_accessor :textfile_data
+  # Conditions to extract time vectors from
+  attr_accessor :conditions
+  # Filename for output csv
+  attr_accessor :csv_filename
+  # Original Presentation Processed Logfile (.txt)
+  attr_reader :textfile
+  # A hash of Onset Vectors keyed by condition
+  attr_reader :vectors
+  
+  def initialize(path, *conditions)
+    @textfile = path
+    @textfile_data = []
+    @conditions = conditions.select {|cond| cond.respond_to? 'to_sym'  }
+    
+    # Add the keys of any hashes (the combined conditions) to the list of 
+    # conditions and add the separate vectors to the list of combined_conditions
+    @combined_conditions = []
+    conditions.select {|cond| cond.respond_to? 'keys' }.each do |cond|
+      cond.keys.collect {|key| @conditions << key }
+      @combined_conditions << cond
+    end
+    
+    raise IOError, "Can't find file #{path}" unless File.exist?(path)
+    File.open(path, 'r').each do |line| 
+      @textfile_data << line.split(/[\,\:\n\r]+/).each { |val| val.strip } 
+    end
+    
+    raise IOError, "Problem reading #{@textfile} - no data found." if @textfile_data.empty?
+
+    if @conditions.empty?
+      raise ScriptError, "Could not set conditions #{conditions}" unless conditions.empty?
+    end
+    
+    setup_logger
+
+  end
+  
+  def condition_vectors
+    return @vectors if @vectors
+    raise ScriptError, "Conditions must be set to extract vectors" if @conditions.empty?
+    vectors = extract_condition_vectors(@conditions)
+    @vectors = zero_and_convert_to_reps(vectors)
+  end
+  
+  def to_csv
+    rows = condition_vectors.values
+    rows = pad_array(rows)
+    
+    output = ""
+    output <<  vectors.keys.join(', ') + "\n"
+    vectors.values.transpose.each do |row|
+      output << row.join(', ') + "\n"
+    end
+    
+    return output
+  end
+  
+  def write_csv(filename)
+    File.open(filename, 'w') { |f| f.puts to_csv }
+    raise ScriptError, "Unable to write #{filename}" unless File.exist?(filename)
+    @csv_filename = filename
+  end
+  
+  def write_mat(prefix)
+    queue = MatlabQueue.new    
+    queue.paths << [
+      Pathname.new(File.join(File.dirname(__FILE__), 'matlab_helpers'))
+    ]
+    
+    raise ScriptError, "Can't find #{@csv_filename}" unless File.exist?(@csv_filename)
+
+    queue << "prepare_onsets_xls( \
+      '#{@csv_filename}', \
+      '#{prefix}', \
+      { #{@conditions.collect {|c| "'#{c}'"}.join(' ') } } \
+    )"
+    
+    queue.run!
+    
+    raise ScriptError, "Problem writing #{prefix}.mat" unless File.exist?(prefix + '.mat')
+    
+    return prefix + '.mat'
+  end
+  
+  # Sort Logfiles by their Modification Time
+  def <=>(other_logfile)
+    File.stat(@textfile).mtime <=> File.stat(other_logfile.textfile).mtime
+  end
+  
+  def self.write_summary(filename = 'tmp.csv', directory = Dir.pwd, grouping = 'version')
+    table = self.summarize_directory(directory)
+    File.open(filename, 'w') do |f| 
+      f.puts Ruport::Data::Grouping(table, :by => grouping).to_csv
+    end
+  end
+    
+  def self.summarize_directory(directory)
+    table = Ruport::Data::Table.new
+    Dir.glob(File.join(directory, '*.txt')).each do |logfile| 
+      # Intialize a logfile without any conditions.
+      lf = Logfile.new(logfile)
+      table << lf.ruport_summary
+      table.column_names = lf.summary_attributes if table.column_names.empty?
+    end
+    return table
+  end
+  
+  # Create an item analysis table containing average response time and 
+  # accuracy for each item over all logfiles.
+  def self.item_analysis(directory)
+    logfiles = Dir.glob(File.join(directory, '*.txt')).collect { |lf| lf = Logfile.new(lf) }
+    all_items = self.item_table(logfiles)
+    item_groups = self.group_items_by(all_items, ['code1', 'pair_type'])
+    summary = self.item_summary(item_groups)
+    
+    return summary
+  end
+  
+  # Create a Ruport::Data::Table of all the response pairs for a given logfile.
+  def response_pairs_table
+    response_pairs = []
+    
+    @textfile_data.each do |line|
+      next if line.empty?
+      header = line.first.gsub(/(\(|\))/, '_').downcase.chomp("_").to_sym
+      if line[1] =~ /(o|n)_\w\d{2}/
+        response_pairs << Ruport::Data::Record.new(response_pair_data(line), :attributes => response_pair_attributes)
+      end
+    end    
+    
+    return response_pairs
+  end
+  
+  # Fields to be included in a Response Pair Table
+  def response_pair_attributes
+    ['enum', 'version', 'ctime', 'pair_type', 'code1', 'time1', 'code2', 'time2', 'time_diff']
+  end
+  
+  def ruport_summary
+    Ruport::Data::Record.new(summary_data, :attributes => summary_attributes )
+  end
+  
+  def summary_attributes
+    ['enum', 'task', 'version', 'ctime'] + @textfile_data[0]
+  end
+  
+  # Combine vectors into a new one (new_misses + old_misses = misses)
+  def combine_vectors(combined_vector_title, original_vector_titles)
+    # Add the combined vectors to the vectors instance variable.
+    condition_vectors[combined_vector_title] = combined_vectors(original_vector_titles)
+    
+    # Add the new condition to @conditions
+    @conditions << combined_vector_title
+  end
+    
+  private
+  
+  def combined_vectors(titles)
+    combined_vector = []
+    puts titles
+    titles.each do |title|
+      if condition_vectors[title]
+        combined_vector << condition_vectors[title]
+      else
+        pp condition_vectors
+        raise "Couldn't find vector called #{title}"
+      end
+    end
+    pp combined_vector
+    return combined_vector.flatten.sort
+  end  
+  
+  # Create a table of all responses to all items.
+  # Pass in an array of #Logfiles
+  def self.item_table(logfiles)
+    table = Ruport::Data::Table.new
+    
+    logfiles.each do |logfile| 
+      logfile.response_pairs_table.collect { |pair| table << pair }
+      table.column_names = logfile.response_pair_attributes if table.column_names.empty?
+    end
+    
+    return table    
+  end
+  
+  def self.item_summary(grouping)
+    item_summary_table = Ruport::Data::Table.new
+    grouping.data.keys.each do |code| 
+      # Create a new copy of the grouping, because the act of summarizing a 
+      # grouping does something destructive to it.
+      current_group = grouping.dup; 
+      samples = current_group.data.values.first.length
+      
+      
+      # Return the summary of codes for each item (in the current case, that's 
+      # <condition>_correct, <condition>_incorrect, and misses)
+      item_records = current_group.subgrouping(code).summary(:code1, 
+        :response_time_average => lambda {|sub| sub.sigma("time_diff").to_f / sub.length}, 
+        :count => lambda {|sub| sub.length}, 
+        :accuracy => lambda {|sub| sub.length / samples.to_f }, 
+        :version => lambda {|sub| sub.data[0][1] }  
+      );
+      
+      # Extract just the first record (correct responses) for the item analysis
+      # as incorrect can be inferred.  Currently, ditch the misses.
+      item_record = item_records.to_a.first
+      
+      # Add the item to the record as a column (the summary doesn't give this
+      # by default because it assumes we already know what we're summarizing.)
+      item_record['code1'] = code; 
+      item_summary_table << item_record
+    end
+    
+    return item_summary_table
+  end
+  
+  def self.group_items_by(table, groupings)
+    Ruport::Data::Grouping.new(table, :by => groupings)
+  end  
+  
+  def response_pair_data(line)
+    enum, task, version = File.basename(@textfile).split("_").values_at(0,3,4)
+    enum = File.basename(enum) unless enum.nil?
+    version = File.basename(version, '.txt') unless version.nil?
+    ctime = File.stat(@textfile).ctime
+    
+    [enum, version, ctime] + line
+    
+  end
+  
+
+  
+  def summary_data
+    enum, task, version = File.basename(@textfile).split("_").values_at(0,3,4)
+    enum = File.basename(enum) unless enum.nil?
+    version = File.basename(version, '.txt') unless version.nil?
+    ctime = File.stat(@textfile).ctime
+    
+    [[enum, task, version, ctime], @textfile_data[1]].flatten
+  end  
+  
+  def extract_condition_vectors(conditions)
+    vectors = {}
+    @conditions
+    @textfile_data.each do |line|
+      next if line.empty?
+      # Headers are written in the Textfile as "New(Correct)".
+      # Convert them to proper condition names - downcase separated by underscores
+      header = pretty_condition(line.first)
+      vector = line[2..-1].collect {|val| val.to_f } if line[2..-1]
+
+      # Make sure this isn't a column line inside the logfile.
+      if line[1] =~ /time/ 
+        # Check if this vector matches any combined conditions.
+        @combined_conditions.each do |vector_group|
+          vector_group.each_pair do |key, vals| 
+            if vals.include?(header)
+              vectors[key] = vectors[key] ? vector.collect{ |timepoint| vectors[key] << timepoint} : vector
+              vectors[key].flatten!
+            end
+          end
+        end
+        
+        # Check if this vector matches a single condition.
+        vectors[header] = vector if @conditions.include?(header);
+      end
+    end
+    
+    # Ensure that the vecotors are in order.
+    vectors.each_value { |vector| vector.sort! }
+    
+    raise ScriptError, "Unable to read vectors for #{@textfile}" if vectors.empty?
+        
+    return vectors
+  end
+  
+  def zero_and_convert_to_reps(vectors)
+    minimum = vectors.values.flatten.min
+    vectors.values.each do |row|
+      row.collect! { |val| (val - minimum) / 2000 }
+    end
+    
+    return vectors
+  end
+  
+  def pad_array(rows)
+    max_length = rows.inject(0) { |max, row| max >= row.length ? max : row.length }
+    rows.each do |row|
+      row[max_length] = nil
+      row.pop
+    end
+  end
+  
+  def pretty_condition(condition)
+    condition.gsub(/(\(|\))/, '_').downcase.chomp("_").to_sym
+  end
+  
+end

data/lib/matlab_helpers/CreateFunctionalVolumeStruct.m

@@ -0,0 +1,6 @@
+function volume_struct = CreateFunctionalVolumeStruct(studypath, image, bold_reps)
+
+volume_struct = cell(bold_reps,1);
+for vol = 1:bold_reps
+	volume_struct{vol} = strcat(studypath, image, ',', int2str(vol));
+end

data/lib/matlab_helpers/import_csv.m

@@ -0,0 +1,32 @@
+function import_csv(fileToRead1)
+%IMPORTFILE(FILETOREAD1)
+%  Imports data from the specified file
+%  FILETOREAD1:  file to read
+
+%  Auto-generated by MATLAB on 24-May-2010 11:01:12
+
+if ~exist(fileToRead1)
+    error('File %s does not exist.', fileToRead1)
+end
+
+% Import the file
+newData1 = importdata(fileToRead1);
+
+% Split Headers
+headers = regexp(newData1.textdata, ',', 'split');
+% headers = headers{1}
+
+% If Matlab FAILED to read an empty vector, assign it.
+% if size(newData1.textdata, 2) ~= size(headers, 2)
+%     newData1.data(:, size(headers,2)) = NaN
+% end
+
+% Create an array of NaN's of proper dimensions (as many rows as data and as many columns as header) and insert the data on top of it.
+data = zeros(size(newData1.data, 1), size(headers,2)) + nan;
+data(1:size(newData1.data,1),1:size(newData1.data,2)) = newData1.data;
+
+% Create new variables in the base workspace from those fields.
+for i = 1:size(headers, 2)
+    assignin('caller', genvarname(headers{i}{1}), data(:,i));
+end
+

data/lib/matlab_helpers/matlab_queue.rb

@@ -0,0 +1,37 @@
+require 'default_logger'
+
+# Maintain and run matlab commands and paths.
+class MatlabQueue
+  include DefaultLogger
+  
+  attr_accessor :paths, :commands, :ml_command
+  attr_reader :success
+  
+  def initialize
+    @paths = []
+    @commands = []
+    @ml_command = "matlab -nosplash -nodesktop"
+    setup_logger
+  end
+  
+  def to_s
+    [
+      @paths.flatten.collect {|path| "addpath(genpath('#{path}'))"},
+      @commands
+    ].flatten.join('; ')
+  end
+  
+  def run!
+    cmd = @ml_command + " -r \"#{ to_s }; exit\" "
+    @success = run(cmd)
+  end
+  
+  def method_missing(m, *args, &block)
+    @commands.send(m, *args, &block)
+  end
+  
+  def add_to_path(*args)
+    args.each { |arg| @paths << arg }
+  end
+  
+end

data/lib/matlab_helpers/prepare_onsets_xls.m

@@ -0,0 +1,30 @@
+function [] = prepare_onsets_xls(csvfile, matfileprefix, conditions)
+%IMPORTFILE(FILETOREAD1)
+%  Imports data from the specified file
+%  FILETOREAD1:  file to read
+
+import_csv(csvfile);
+
+for i = 1:length(conditions)
+	condition = conditions{i};
+	condition_onsets = eval(condition);
+		
+	% Strip NaN's, but leave one nan if vector is empty (SPM's preference).
+	condition_onsets = condition_onsets(find(~isnan(condition_onsets)));
+	
+    % Allow for conditions called 'misses' to be dropped from onsets.
+    if length(condition_onsets) == 0;
+		if ~strcmp(condition, 'misses')
+            condition_onsets=[nan];
+        else
+            continue
+        end
+	end
+	
+	% Format cell array for SPM's multiple conditions
+	names{i} = condition;
+	onsets{i} = condition_onsets;
+	durations{i} = [0];
+end
+
+save([matfileprefix,'.mat'],'names','onsets', 'durations');

data/lib/rpipe.rb

@@ -0,0 +1,254 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), 'custom_methods'))
+
+require 'rubygems'
+require 'yaml'
+require 'ftools'
+require 'fileutils'
+require 'pathname'
+require 'tmpdir'
+require 'erb'
+require 'log4r'
+require 'popen4'
+require 'core_additions'
+require 'metamri/core_additions'
+
+# prevent zipping in FSL programs
+ENV['FSLOUTPUTTYPE'] = 'NIFTI'
+
+require 'default_logger'
+require 'global_additions'
+
+class JobStep
+	
+	COLLISION_POLICY = :panic # options -- :panic, :destroy, :overwrite
+	
+	attr_accessor :subid, :rawdir, :origdir, :procdir, :statsdir, :spmdir, :collision_policy, :libdir
+	
+	# Intialize with two configuration option hashes - workflow_spec and job_spec
+	def initialize(workflow_spec, job_spec)
+		# allow jobspec to override the workflow spec
+		@subid        = job_spec['subid']       || workflow_spec['subid']
+		@rawdir       = job_spec['rawdir']      || workflow_spec['rawdir']
+		@origdir      = job_spec['origdir']     || workflow_spec['origdir']
+		@procdir      = job_spec['procdir']     || workflow_spec['procdir']
+		@statsdir     = job_spec['statsdir']    || workflow_spec['statsdir']
+		@spmdir       = job_spec['spmdir']      || workflow_spec['spmdir']
+		@scans        = job_spec['scans']       || workflow_spec['scans']
+		@scan_labels  = job_spec['scan_labels'] || workflow_spec['scan_labels'] 
+		@collision_policy = (job_spec['collision'] || workflow_spec['collision'] || COLLISION_POLICY).to_sym
+		@method = job_spec['method']
+		include_custom_methods(@method)
+		@libdir = File.dirname(Pathname.new(__FILE__).realpath)
+		
+		job_requires 'subid'
+	end
+	
+	# Dynamically load custom methods for advanced processing.
+	def include_custom_methods(module_name)
+		if module_name.nil? or ['default','wadrc'].include?(module_name)
+			# do nothing, use default implementation
+		else
+			require module_name
+			extend self.class.const_get(module_name.dot_camelize)
+		end
+	end
+	
+	# Setup directory path according to collision policy.
+	def setup_directory(path, logging_tag)
+		if File.exist?(path)
+			if @collision_policy == :destroy
+				puts "#{logging_tag} :: Deleting directory #{path}"
+				FileUtils.rm_rf(path)
+				FileUtils.mkdir_p(path)
+			elsif @collision_policy == :overwrite
+				puts "#{logging_tag} :: Overwriting inside directory #{path}"
+			else
+				raise(IOError, "Directory already exists, exiting: #{path}")
+			end
+		else
+			puts "#{logging_tag} :: Creating new directory #{path}"
+			FileUtils.mkdir_p(path)
+		end
+	end
+	
+	# Check for required keys in instance variables.
+  def job_requires(*args)
+    check_instance_vars *args do |missing_vars|
+      error = "
+      Warning: Misconfiguration detected.
+      You are missing the following required variables from your spec file:
+      #{missing_vars.collect { |var| "  - #{var} \n" } }
+      "
+      
+      puts error
+      raise ScriptError, "Missing Vars: #{missing_vars.join(", ")}"
+    end
+  end
+  
+  def check_instance_vars(*args)
+    undefined_vars = []
+    args.each do |arg|
+      unless instance_variable_defined?("@" + arg) && eval("@" + arg).nil? == false
+        undefined_vars << arg
+      end
+    end
+    
+    unless undefined_vars.size == 0
+      yield undefined_vars
+    end
+  end
+  
+
+  	
+end
+
+
+
+########################################################################################################################
+# A class for performing initial reconstruction of both functional and anatomical MRI scan acquisitions.
+# Uses AFNI to convert from dicoms to 3D or 4D nifti files, initial volume stripping, and slice timing correction.
+# Currently, supports dicoms or P-Files.
+class Reconstruction < JobStep
+	require 'default_methods/default_recon'
+	include DefaultRecon
+	
+	VOLUME_SKIP = 3 # number of volumes to strip from beginning of functional scans.
+	
+	attr_accessor :scans, :volume_skip
+	
+	# Instances are initialized with a properly configured hash containing all the information needed to drive
+	# reconstruction tasks.	 This hash is normally generated with a Pipe object.
+	def initialize(workflow_spec, recon_spec)
+		super(workflow_spec, recon_spec)
+		raise ScriptError, "At least one scan must be specified." if @scans.nil?
+		@volume_skip = recon_spec['volume_skip'] || VOLUME_SKIP
+		
+		job_requires 'rawdir', 'origdir', 'scans'
+	end
+
+end
+###############################################	 END OF CLASS	 #########################################################
+
+
+
+
+########################################################################################################################
+# A class for performing spatial preprocessing steps on functional MRI data in preparation for first level stats.
+# Preprocessing includes setting up output directories, linking all appropriate data, customizing a preconfigured spm
+# job, running the job, calculating withing scan motion derivatives, and finally checking for excessive motion.
+# The spm job should normally include tasks for realignment, normalization, and smoothing. 
+class Preprocessing < JobStep
+	require 'default_methods/default_preproc'
+	include DefaultPreproc
+
+	MOTION_THRESHOLD = 1 # maximum allowable realignment displacement in any direction
+
+	attr_accessor :tspec, :motion_threshold, :bold_reps
+
+	# Initialize instances with a hash normally generated by a Pipe object.
+	def initialize(workflow_spec, preproc_spec)
+		super(workflow_spec, preproc_spec)
+		@tspec = preproc_spec['template_spec']
+		@motion_threshold = preproc_spec['motion_threshold'] || MOTION_THRESHOLD
+		@bold_reps = preproc_spec['bold_reps']
+		
+		job_requires 'origdir', 'procdir'
+	end
+
+end
+###############################################	 END OF CLASS	 #########################################################
+
+
+
+
+
+########################################################################################################################
+# A class used to compute the first level stats for a functional MRI visit data set.
+# Currently very incomplete, any ideas for other data/attributes we need here?
+class Stats < JobStep
+	require 'default_methods/default_stats'
+	include DefaultStats
+	
+	attr_accessor :statsdir, :tspec, :onsetsfiles, :responses, :regressorsfiles, :bold_reps, :conditions
+	
+	# Initialize instances with a hash normally generated by a Pipe object.
+	def initialize(workflow_spec, stats_spec)
+		super(workflow_spec, stats_spec)
+		@tspec = stats_spec['template_spec']
+		@onsetsfiles = stats_spec['onsetsfiles']
+		@responses = stats_spec['responses']
+		@regressorsfiles = stats_spec['regressorsfiles']
+		@bold_reps = stats_spec['bold_reps']
+		@conditions = stats_spec['conditions'] || workflow_spec['conditions']
+		
+		job_requires 'bold_reps', 'conditions', 'procdir'
+	end
+	
+end
+###############################################	 END OF CLASS	 #########################################################
+
+
+
+
+
+########################################################################################################################
+# An object that drives all the other pipeline classes in this module.	Once initialized, an array of instances of each
+# class are available to run segments of preprocessing.
+class RPipe
+	
+	include Log4r
+	include DefaultLogger
+	
+	attr_accessor :recon_jobs, :preproc_jobs, :stats_jobs, :workflow_spec
+	
+	libdir = File.expand_path(File.dirname(__FILE__))
+	
+	# Initialize an RPipe instance by passing it a pipeline configuration driver.
+	# Drivers contain a list of entries, each of which contains all the 
+	# information necessary to create an instance of the proper object that 
+	# executes the job.	Details on the formatting of the yaml drivers including examples will be
+	# provided in other documentation.
+	#
+	# The driver may be either a Hash or a yaml configuration file.
+	
+	def initialize(driver)	
+		@recon_jobs = []
+		@preproc_jobs = []
+		@stats_jobs = []
+		
+		# A driver may be either a properly configured hash or a Yaml file containing
+		# the configuration.  
+		@workflow_spec = driver.kind_of?(Hash) ? driver : read_driver_file(driver)
+		
+    setup_logger
+    
+    jobs = @workflow_spec['jobs']
+		jobs.each do |job_params|
+			@recon_jobs   << Reconstruction.new(@workflow_spec, job_params) if job_params['step'] == 'reconstruct'
+			@preproc_jobs << Preprocessing.new(@workflow_spec, job_params)  if job_params['step'] == 'preprocess'
+			@stats_jobs   << Stats.new(@workflow_spec, job_params)          if job_params['step'] == 'stats'
+		end
+		
+		def jobs
+		  [@recon_jobs, @preproc_jobs, @stats_jobs].flatten
+	  end
+	  
+	end
+	
+	# Reads a YAML driver file, parses it with ERB and returns the Configuration Hash.
+	# Raises an error if the file is not found in the file system.
+	def read_driver_file(driver_file)
+		raise(IOError, "Driver file not found: #{driver_file}") unless File.exist?(driver_file)
+		YAML.load(ERB.new(File.read(driver_file)).result) 
+  end
+  
+  private
+  
+  # To compare jobs look at their configuration, not ruby object identity.
+  def ==(other_rpipe)
+    @workflow_spec == other_rpipe.workflow_spec
+  end
+	
+end
+###############################################	 END OF CLASS	 #########################################################