fcsparse 0.1.0

data/lib/fcsparse/fcsconst.rb

@@ -0,0 +1,93 @@
+#--
+# /* ***** BEGIN LICENSE BLOCK *****
+#  * 
+#  * Copyright (c) 2012 Colin J. Fuller
+#  * 
+#  * Permission is hereby granted, free of charge, to any person obtaining a copy
+#  * of this software and associated documentation files (the Software), to deal
+#  * in the Software without restriction, including without limitation the rights
+#  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  * copies of the Software, and to permit persons to whom the Software is
+#  * furnished to do so, subject to the following conditions:
+#  * 
+#  * The above copyright notice and this permission notice shall be included in
+#  * all copies or substantial portions of the Software.
+#  * 
+#  * THE SOFTWARE IS PROVIDED AS IS, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+#  * SOFTWARE.
+#  * 
+#  * ***** END LICENSE BLOCK ***** */
+#++
+
+##
+# Contains all classes and constants for parsing FCS v3.x formatted files.
+#
+module FCSParse
+  
+  #
+  # Contains constants associated with the fcs file format, such as byte offsets
+  # and parameter names.
+  # 
+  # The first character of each constant name specifies the section of the fcs file
+  # to which it is relevant.  (H = header, T = text, etc.)
+  # 
+  # @author Colin J. Fuller
+  #
+  
+  #byte offsets to the start and end of the version string
+  H_VersionStart = 0
+  H_VersionEnd = 5
+  
+  #length of the block specifying the offsets of text, data, analysis sections
+  H_OffsetBlockLength = 8
+  
+  #offsets to the start and end of the text section offset
+  H_TextBlockOffsetStart = 10
+  H_TextBlockOffsetEnd = 18
+  
+  #offsets to the start and end of the data section offset
+  H_DataBlockOffsetStart = 26
+  H_DataBlockOffsetEnd = 34
+  
+  #offsets to the start and end of the analysis section offset
+  H_AnalysisBlockOffsetStart = 42 
+  H_AnalysisBlockOffsetEnd = 50
+  
+  #keyword specifying offset to supplementary text section
+  T_SupplTextStartKeyword = :$BEGINSTEXT
+  T_SupplTextEndKeyword = :$ENDSTEXT
+  
+  #keyword specifying offset to analysis section
+  T_AnalysisStartKeyword = :$BEGINANALYSIS
+  T_AnalysisEndKeyword = :$BEGINANALYSIS
+  
+  #keyword specifying offset to data section
+  T_DataStartKeyword = :$BEGINDATA
+  T_DataEndKeyword = :$ENDDATA
+  
+  #keyword specifying the data mode
+  T_ModeKeyword = :$MODE
+  
+  #keyword specifying the data type (e.g. integer, float, etc)
+  T_DatatypeKeyword = :$DATATYPE
+  
+  #keyword specifying byte order and value when little endian
+  T_ByteorderKeyword = :$BYTEORD
+  T_LittleEndianByteorder = "1,2,3,4"
+  
+  #keyword specifying the number of parameters measured per event
+  T_ParameterCountKeyword = :$PAR
+  
+  #keyword specifying total number of events
+  T_EventCountKeyword = :$TOT
+  
+  #regular expressions matching names and ranges of all parameters
+  T_ParameterNameKeywordRegex = /P(\d+)N/
+  T_ParameterRangeKeywordRegex = /P(\d+)R/
+
+end

data/lib/fcsparse/fcsevent.rb

@@ -0,0 +1,215 @@
+#--
+# /* ***** BEGIN LICENSE BLOCK *****
+#  * 
+#  * Copyright (c) 2012 Colin J. Fuller
+#  * 
+#  * Permission is hereby granted, free of charge, to any person obtaining a copy
+#  * of this software and associated documentation files (the Software), to deal
+#  * in the Software without restriction, including without limitation the rights
+#  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  * copies of the Software, and to permit persons to whom the Software is
+#  * furnished to do so, subject to the following conditions:
+#  * 
+#  * The above copyright notice and this permission notice shall be included in
+#  * all copies or substantial portions of the Software.
+#  * 
+#  * THE SOFTWARE IS PROVIDED AS IS, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+#  * SOFTWARE.
+#  * 
+#  * ***** END LICENSE BLOCK ***** */
+#++
+
+require 'fcsparse/fcsconst'
+
+##
+# Contains all classes and constants for parsing FCS v3.x formatted files.
+#
+module FCSParse
+  
+  ##
+  # Class representing a single parameter and its value in a single event.
+  # 
+  # @author Colin J. Fuller
+  # 
+  class FCSParam
+    
+    ##
+    # Create a new parameter with specified information
+    # 
+    # @param name   the name of the parameter
+    # @param description  a longer description of the parameter
+    # @param value  the value of the parameter
+    # @param limit  the maximum value that the parameter can take
+    # 
+    def initialize(name, description, value, limit)
+      
+      @name = name
+      @description = description
+      @value = value
+      @limit = limit
+      
+    end
+    
+    attr_accessor :name, :description, :value, :limit
+    
+  end
+  
+  
+  ##
+  # Class representing a single FCS-encoded event.
+  # 
+  # @author Colin J. Fuller
+  # 
+  class FCSEvent
+    
+    #default delimiter for printing output
+    DefaultDelimiter = ","
+    
+    private_class_method :new
+    
+    def initialize
+      
+      @values = Hash.new
+      
+    end
+    
+    
+    ##
+    # Creates a new FCSEvent from the specified information.
+    # 
+    # @param [String] event_data_string   the raw data corresponding to the
+    #         entire event from the fcs file
+    # @param [String] event_format_string a string suitable for use with String#unpack
+    #         that can decode the raw data.
+    # @param [Hash] parameter_info_hash   a hash containing at minimum the parameters
+    #         from the text section specifying the names and ranges of the parameters
+    #         keys should be the parameter names from the fcs file format converted
+    #         to symbols ($ included), and values should be a string corresponding
+    #         to the value of the parameter from the fcs file.
+    # @return [FCSEvent] an FCSEvent that has been created by parsing the raw data.
+    # 
+    def self.new_with_data_and_format(event_data_string, event_format_string, parameter_info_hash)
+    
+      data_points = event_data_string.unpack(event_format_string)
+      
+      parameter_names = Hash.new
+      parameter_limits = Hash.new
+      
+      parameter_info_hash.each_key do |k|
+        
+        matchobj = k.to_s.match(T_ParameterNameKeywordRegex)
+        
+        
+        if matchobj then
+          
+          parameter_names[matchobj[1].to_i] = parameter_info_hash[k]
+          
+        end
+          
+
+        matchobj = k.to_s.match(T_ParameterRangeKeywordRegex)
+        
+        if matchobj then
+          
+          parameter_limits[matchobj[1].to_i] = parameter_info_hash[k]
+
+        end
+    
+      end
+      
+      ordered_indices = parameter_names.keys.sort
+            
+      event = new      
+      
+      data_points.each_with_index do |e, i|
+      
+        param = FCSParam.new(parameter_names[ordered_indices[i]], nil, e, parameter_limits[ordered_indices[i]])
+        
+        event[param.name] = param
+        
+      end
+      
+      event
+              
+    end    
+    
+    ##
+    # Gets a named parameter associated with the event.
+    # 
+    # @param [String] parameter_name  the name of the parameter to retrieve; this should be 
+    #                 exactly the name specified for the parameter in the text
+    #                 section of the fcs file
+    # @return [FCSParam]  an FCSParam object that holds the information about the named parameter.
+    # 
+    def [](parameter_name)
+      
+      @values[parameter_name]
+      
+    end
+    
+    ##
+    # Sets a named parameter associated with the event.
+    # @param [String] parameter_name  the name of the parameter to retrieve; this should be 
+    #                 exactly the name specified for the parameter in the text
+    #                 section of the fcs file
+    # @param [FCSParam] value   an FCSParam object that holds the information about the named parameter.
+    # 
+    def []=(parameter_name, value)
+      
+      @values[parameter_name]= value
+      
+    end
+    
+    ##
+    # Gets the names of the parameters associated with this event in alphabetical
+    # order as a string, delimited by the supplied delimiter.
+    # 
+    # @param [String] delimiter a String containing the desired delimiter.
+    # @return [String]  a String containing delimited alphabetized parameter names.
+    # 
+    def names_to_s_delim(delimiter)
+      
+      all_param_names = @values.keys.sort
+      
+      all_param_names.join(delimiter)
+      
+    end
+    
+    ##
+    # Gets the values of the parameters associated with this event ordered 
+    # alphabetically by the parameter names (i.e. in the same order as when calling
+    # {#names_to_s_delim}), delimited by the supplied delimiter.
+    # 
+    # @param [String]delimiter a String containing the desired delimiter.
+    # @return [String]  a String containing delimited ordered parameter values.
+    #
+    def to_s_delim(delimiter)
+            
+      all_param_names = @values.keys.sort
+      
+      all_param_values = all_param_names.map{|e| @values[e].value}
+      
+      all_param_values.join(delimiter)
+      
+    end
+    
+    ##
+    # Converts the event to a string representation.  This is the same as calling
+    # {#to_s_delim} with the delimiter set to {FCSEvent::DefaultDelimiter}.
+    # 
+    # @return [String]  a String containing delimited ordered parameter values.
+    #
+    def to_s
+      
+      to_s_delim(DefaultDelimiter)
+      
+    end
+
+  end
+  
+end

data/lib/fcsparse.rb

@@ -0,0 +1,363 @@
+#--
+# /* ***** BEGIN LICENSE BLOCK *****
+#  * 
+#  * Copyright (c) 2012 Colin J. Fuller
+#  * 
+#  * Permission is hereby granted, free of charge, to any person obtaining a copy
+#  * of this software and associated documentation files (the Software), to deal
+#  * in the Software without restriction, including without limitation the rights
+#  * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+#  * copies of the Software, and to permit persons to whom the Software is
+#  * furnished to do so, subject to the following conditions:
+#  * 
+#  * The above copyright notice and this permission notice shall be included in
+#  * all copies or substantial portions of the Software.
+#  * 
+#  * THE SOFTWARE IS PROVIDED AS IS, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+#  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+#  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+#  * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+#  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+#  * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+#  * SOFTWARE.
+#  * 
+#  * ***** END LICENSE BLOCK ***** */
+#++
+
+require 'fcsparse/fcsconst'
+require 'fcsparse/fcsevent'
+
+##
+# Contains all classes and constants for parsing FCS v3.x formatted files.
+# 
+module FCSParse
+  
+  ##
+  # A class representing an FCS-encoded file.  Has methods to parse the data,
+  # manage it, and convert it to human-readable format.
+  # 
+  # Currently reads FCS v3.x files.  Files must have data encoded in list mode,
+  # and data points must be in float or double format (this will change eventually to support
+  # all formats in the specification).
+  # 
+  # Analysis sections of the file are ignored.
+  # 
+  # Data can be written to delimited text files, and metadata to plain text files,
+  # or the objects containing the data can be used by other code to process
+  # the data further.
+  # 
+  # @author Colin J. Fuller
+  # 
+  class FCSFile
+    
+    MetadataExtension = ".meta.txt"
+    DataExtension = ".data.csv"
+    DataDelimiter = FCSEvent::DefaultDelimiter
+    SupportedVersions = ["FCS3.0", "FCS3.1"]
+        
+    attr_accessor :filename, :events
+    
+    
+    ##
+    # Generates a new FCSFile object from the specified file.
+    # 
+    # Calling this will read the entire file into memory but will not parse it.
+    # 
+    # @param [String] filename  the filename of the FCS-encoded file (with path as required to locate it)
+    # @return [FCSFile]   a new FCSFile object initialized with the contents of the file.
+    # 
+    def self.new_from_file(filename)
+      
+      fcsfile = nil
+      
+      File.open(filename) do |f|
+        fcsfile = self.new(f.read)
+      end
+      
+      fcsfile.filename = filename
+      
+      fcsfile
+      
+    end
+    
+    ##
+    # Generates a new FCSFile from the specified FCS-encoded data string.
+    # 
+    # @param [String] file_string   a String containing an FCS-encoded dataset.
+    def initialize(file_string)
+      @data = file_string
+      @keywords = Hash.new
+    end
+
+    def read_header
+
+      version_string= @data[H_VersionStart..H_VersionEnd]
+
+      unless SupportedVersions.include?(version_string) then
+        raise "Unable to read this FCS format: " + version_string
+      end
+      
+
+      @version = version_string[3..5].to_f
+      
+      @text_offsets = Array.new
+      @data_offsets = Array.new
+      @analysis_offsets = Array.new
+      
+      @text_offsets << @data[H_TextBlockOffsetStart, H_OffsetBlockLength].to_i
+      @text_offsets << @data[H_TextBlockOffsetEnd, H_OffsetBlockLength].to_i
+      
+      @data_offsets << @data[H_DataBlockOffsetStart, H_OffsetBlockLength].to_i
+      @data_offsets << @data[H_DataBlockOffsetEnd, H_OffsetBlockLength].to_i
+      
+      @analysis_offsets << @data[H_AnalysisBlockOffsetStart, H_OffsetBlockLength].to_i
+      @analysis_offsets << @data[H_AnalysisBlockOffsetEnd, H_OffsetBlockLength].to_i
+            
+    end
+    
+    def parse_text_segment_with_bounds(lower_bound, upper_bound)
+      
+      token_queue = Array.new
+      
+      delimiter = @data[lower_bound]
+      
+      token_start = lower_bound + 1
+      
+      (lower_bound+1).upto(upper_bound) do |i|
+        
+        if @data[i] == delimiter and not @data[i-1] == delimiter and not @data[i+1] == delimiter then
+          
+          token_queue << @data[token_start...i]
+          token_start = i+1
+          
+        end
+        
+      end
+      
+      token_queue.each_slice(2) do |kv|
+        
+        @keywords[kv[0].upcase.to_sym]= kv[1]
+        
+      end
+      
+      
+    
+    end    
+    
+    def parse_text_segment
+      
+      parse_text_segment_with_bounds(@text_offsets[0].to_i, @text_offsets[1].to_i)
+      
+      suppl_text_start = @keywords[T_SupplTextStartKeyword].to_i
+      suppl_text_end = @keywords[T_SupplTextEndKeyword].to_i
+      
+      unless suppl_text_start == 0 and suppl_text_end == 0 then
+        parse_text_segment_with_bounds(suppl_text_start, suppl_text_end)
+      end
+      
+      @data_offsets[0] = @keywords[T_DataStartKeyword].to_i
+      @data_offsets[1] = @keywords[T_DataEndKeyword].to_i
+      
+      @analysis_offsets[0] = @keywords[T_AnalysisStartKeyword].to_i
+      @analysis_offsets[1] = @keywords[T_AnalysisEndKeyword].to_i
+      
+      unless @keywords[T_ModeKeyword] == "L" then
+        raise "Only list mode is supported for the data block."
+      end
+      
+    end
+    
+    def parse_data_segment
+      
+      event_format = construct_event_format_string
+      
+      event_count = @keywords[T_EventCountKeyword].to_i
+      
+      bytes_per_event = ((@data_offsets[1] - @data_offsets[0] + 1)*1.0/event_count).to_i
+            
+      @events = Array.new
+            
+      0.upto(event_count -1) do |e|
+        
+        offset = @data_offsets[0] + bytes_per_event*e
+        
+        event_string = @data[offset, bytes_per_event]
+        
+        event = FCSEvent.new_with_data_and_format(event_string, event_format, @keywords)
+        
+        @events << event
+                
+      end
+      
+    end
+    
+    def construct_event_format_string
+      
+      datatype = @keywords[T_DatatypeKeyword]
+      
+      is_little_endian = (@keywords[T_ByteorderKeyword] == T_LittleEndianByteorder)
+            
+      formatchar = case datatype
+      
+      when 'I'
+        
+        nil
+         
+      when 'F'
+        
+        if is_little_endian then
+          
+          'e'
+          
+        else
+          
+          'g'
+          
+        end
+        
+      when 'D'
+        
+        if is_little_endian then
+          
+          'E'
+          
+        else
+          
+          'G'
+          
+        end
+        
+      when 'A'
+        
+        nil
+        
+      end
+        
+      unless formatchar then
+        raise "Integer and string data not yet supported."
+      end
+            
+      parameter_count = @keywords[T_ParameterCountKeyword].to_i
+            
+      formatchar*parameter_count
+      
+    end
+    
+    ##
+    # Gets the metadata (that is all the information in the text block of the
+    # fcs file) as a string, where there is one key, value pair per line, separated
+    # by the characters " => "
+    # 
+    # @return [String]    a String containing all the metadata
+    def get_metadata_string
+      str = ""
+      @keywords.keys.map{|e| e.to_s}.sort.each do |k|
+        str << k.to_s + " => " + @keywords[k.to_sym].to_s + "\n"
+      end
+      str
+    end
+    
+    ##
+    # Prints the metadata string returned by {#get_metadata_string}.
+    # 
+    def print_metadata
+
+      puts get_metadata_string
+      
+      nil
+      
+    end
+    
+    ##
+    # Writes the metadata and data from the FCS-formatted file to disk in human
+    # readable format.
+    # 
+    # The metadata is written as a text file (formatted by 
+    # {#get_metadata_string}), in the same location as the input FCS file and
+    # with the same name except for an extra extension specified in 
+    # {FCSFile::MetadataExtension}.
+    # 
+    # The data, delimited by the data delimiter specified
+    # as {FCSFile::DataDelimiter}, one row per event, is written to a text file
+    # in the same location as the input FCS file and with the same name except
+    # for an extra extension specified in {FCSFile::DataExtension}.
+    # 
+    # @param [Boolean] write_header_row an optional parameter specifying whether
+    # the data file should have a header row with the name of each column's 
+    # parameter.  Defaults to true.
+    # 
+    def write_metadata_and_data(write_header_row = true)
+      
+      meta_filename = @filename + MetadataExtension
+      
+      data_filename = @filename + DataExtension
+      
+      File.open(meta_filename, "w") do |f|
+        
+        f.write(get_metadata_string)
+        
+      end
+      
+      File.open(data_filename, "w") do |f|
+        
+        if write_header_row then
+          f.puts(@events[0].names_to_s_delim(DataDelimiter))
+        end
+        
+        @events.each do |e|
+          
+          f.puts(e.to_s_delim(DataDelimiter))
+        
+        end
+        
+      end
+      
+      nil
+      
+    end
+    
+    ##
+    # Parses the raw data loaded in the FCSFile to metadata, events, and parameters.
+    # 
+    # Erases the raw data from the FCSFile object after parsing is complete.
+    # 
+    def parse
+      read_header
+      parse_text_segment
+      parse_data_segment
+      @data = nil
+    end
+
+    private :read_header, :parse_text_segment_with_bounds, :parse_text_segment, :parse_data_segment, :construct_event_format_string
+
+  end
+  
+  
+  ##
+  # Processes a specified FCS-formatted file, and writes human-readable output
+  # to disk in the format specified by {FCSFile#write_metadata_and_data}.
+  # 
+  # @param [String] filename  the filename of the FCS-encoded file (with path as required to locate it)
+  # @param [Boolean] data_header_row an optional parameter specifying whether
+  # the data file should have a header row with the name of each column's 
+  # parameter.  Defaults to true.
+  # 
+  def self.process_file(filename, data_header_row = true)
+    
+    fcsfile = FCSFile.new_from_file(filename)
+    fcsfile.parse
+    fcsfile.write_metadata_and_data(data_header_row)
+    
+    nil
+    
+  end 
+    
+
+
+end
+
+if __FILE__ == $0 then
+  
+  FCSParse.process_file(ARGV[0], false)
+  
+end

metadata

@@ -0,0 +1,68 @@
+--- !ruby/object:Gem::Specification 
+name: fcsparse
+version: !ruby/object:Gem::Version 
+  hash: 1322991910812937300
+  prerelease: 
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Colin J. Fuller
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-06-18 00:00:00 Z
+dependencies: []
+
+description: Parses flow cytometry FCS v3.x files and allows output to plain (delimited) text.
+email: cjfuller@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- lib/fcsparse.rb
+- lib/fcsparse/fcsconst.rb
+- lib/fcsparse/fcsevent.rb
+homepage: http://code.google.com/p/fcsparse/
+licenses: 
+- MIT
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 2002549777813010636
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 2002549777813010636
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.8.12
+signing_key: 
+specification_version: 3
+summary: Parser for FCS v3.x file format
+test_files: []
+
+has_rdoc: