gfa 0.2.0 → 0.4.0

data/lib/gfa/graph.rb

@@ -2,7 +2,6 @@ require 'rgl/adjacency'
 require 'rgl/implicit'
 
 class GFA
-   
   ##
   # Generates a RGL::ImplicitGraph object describing the links in the GFA.
   # The +opts+ argument is a hash with any of the following key-value pairs:
@@ -22,9 +21,145 @@ class GFA
   def adjacency_graph(opts = {})
     implicit_graph(opts).to_adjacency
   end
-   
+
+  ##
+  # Extracts the subset of records associated to +segments+, which is an Array
+  # with values of any class in: Integer (segment index),
+  # String or GFA::Field::String (segment names), or GFA::Record::Segment.
+  #
+  # +degree+ indicates the maximum degree of separation between the original
+  # segment set and any additional segments. Use 0 to include only the segments
+  # in the set. Use 1 to include those, the records linking to them, and the
+  # additional segments linked by those records. Use any integer greater than 1
+  # to prompt additional rounds of greedy graph expansion.
+  #
+  # If +headers+, it includes all the original headers. Otherwise it only
+  # only includes the version header (might be inferred).
+  #
+  # All comments are ignored even if originally parsed. Walks are currently
+  # ignored too. If the current GFA object doesn't have an index, it builds one
+  # and forces +index: true+. The output object inherits all options.
+  def subgraph(segments, degree: 1, headers: true)
+    # Prepare objects
+    unless opts[:index]
+      opts[:index] = true
+      rebuild_index!
+    end
+    gfa = GFA.new(opts)
+    segments =
+      segments.map do |i|
+        i.is_a?(GFA::Record::Segment) ? i :
+          segment(i) or raise "Cannot find segment: #{i}"
+      end
+
+    # Headers
+    if headers
+      self.headers.set.each { |record| gfa << record }
+    else
+      gfa << GFA::Record::Header.new("VN:Z:#{gfa_version}")
+    end
+
+    # Original segments
+    segments.each { |segment| gfa << segment }
+
+    # Expand graph
+    linking, edges = linking_records(gfa.segments, degree: degree)
+    linking += internally_linking_records(segments, edges)
+    linking.each { |record| gfa << record }
+
+    # Return
+    gfa
+  end
+
+  ##
+  # Finds all the records linking to any segments in +segments+, a
+  # GFA::RecordSet::SegmentSet object, and expands to links with up to
+  # +degree+ degrees of separation
+  #
+  # It only evaluates the edges given in the +edges+ Array of GFA::Record
+  # values. If +edges+ is +nil+, it uses the full set of edges in the gfa.
+  # Edge GFA::Record objects can be of type Link, Containment, Jump, or Path
+  #
+  # If +_ignore+ is passed, it ignores this number of segments at the beginning
+  # of the +segments+ set (assumes they have already been evaluated). This is
+  # only used for internal heuristics
+  #
+  # Returns an Array of with two elements:
+  # 0. An array of GFA::Record objects with all the identified linking records
+  # 1. An array of GFA::Record objects with all edges that were not identified
+  #
+  # IMPORTANT NOTE 1: The object +segments+ will be modified to include all
+  # linked segments. If you don't want this behaviour, please make sure to pass
+  # a duplicate of the object instead.
+  #
+  # IMPORTANT NOTE 2: The list of linking records may not comprehensively
+  # include all records linking the identified expanded segment set. To ensure
+  # a consistent set is identified, use:
+  # linking, edges = gfa.linking_records(segments)
+  # linking += gfa.internally_linking_records(segments, edges)
+  # 
+  def linking_records(segments, degree: 1, edges: nil, _ignore: 0)
+    unless segments.is_a? GFA::RecordSet::SegmentSet
+      raise "Unrecognised class: #{segments.class}"
+    end
+
+    # Gather edges to evaluate
+    edges ||= all_edges
+    return [[], edges] if degree <= 0
+
+    # Links, Containments, Jumps (from, to) and Paths (segment_names)
+    linking = []
+    eval_set = _ignore == 0 ? segments.set : segments.set[_ignore..]
+    edges.delete_if do |record|
+      if eval_set.any? { |segment| record.include? segment }
+        linking << record
+        true  # Remove from the edge set to speed up future recursions
+      else
+        false # Keep it, possibly linking future recursions 
+      end
+    end
+
+    # Recurse and return
+    if degree >= 1
+      pre = segments.size
+
+      # Add additional linked segments
+      linking.each do |record|
+        record.segments(self).each do |other_seg|
+          segments << other_seg unless segments[other_seg.name]
+        end
+      end
+
+      # Recurse only if new segments were discovered
+      if segments.size > pre
+        $stderr.puts "- Recursion [#{degree}]: " \
+                     "#{pre} -> #{segments.size}\t(#{edges.size})"
+        linking +=
+          linking_records(
+            segments,
+            degree: degree - 1, edges: edges, _ignore: pre
+          )[0]
+      end
+    end
+    [linking, edges]
+  end
+
+  def internally_linking_records(segments, edges)
+    $stderr.puts '- Gathering internally linking records'
+    segments = Hash[segments.set.map { |i| [i.name.value, true]}]
+    edges.select { |record| record.segment_names_a.all? { |s| segments[s] } }
+  end
+
+  ##
+  # Returns an array of GFA::Record objects including all possible edges
+  # from the GFA. I.e., all links, jumps, containments, and paths.
+  def all_edges
+    edge_t = %i[Link Jump Containment Path]
+    edges = edge_t.flat_map { |t| records[t].set } if edges.nil?
+  end
+
   private
-   
+
     def segment_names_with_orient
       segments.flat_map do |s|
         %w[+ -].map { |orient| GFA::GraphVertex.idx(s, orient) }
@@ -57,7 +192,7 @@ class GFA
       opts
     end
 
-    def rgl_implicit_adjacent_iterator(x,b,opts)
+    def rgl_implicit_adjacent_iterator(x, b, opts)
       links.each do |l|
         if l.from?(x.segment, x.orient)
           orient = opts[:orient] ? l.to_orient : nil

data/lib/gfa/parser.rb

@@ -4,43 +4,99 @@ class GFA
   # Class-level
   MIN_VERSION = '1.0'
   MAX_VERSION = '1.2'
-   
-  def self.load(file)
-    gfa = GFA.new
-    fh = File.open(file, 'r')
-    fh.each { |ln| gfa << ln }
-    fh.close
+
+  ##
+  # Load a GFA object from a gfa +file+ with options +opts+:
+  # - index: If the  records should be indexed as loaded (default: true)
+  # - index_id: If the records should also be index by ID (default: false)
+  # - comments: If the comment records should be saved (default: false)
+  # - line_range: Two-integer array indicating the first and last lines to read
+  #   (default: nil, read the entire file)
+  def self.load(file, opts = {})
+    gfa = GFA.new(opts)
+    read_records(file, opts) do |record|
+      gfa << record
+    end
     gfa
   end
-   
+
+  def self.read_records(file, opts = {})
+    rng = opts[:line_range]
+    File.open(file, 'r') do |fh|
+      lno = -1
+      fh.each do |ln|
+        lno += 1
+        next if !rng.nil? && (lno < rng[0] || lno > rng[1])
+        next if !opts[:comments] && ln[0] == '#'
+
+        yield(GFA::Record[ln])
+      end
+    end
+  end
+
+  ##
+  # Load a GFA object from a gfa +file+ in parallel using +thr+ threads,
+  # and the same +opts+ supported by +load+. Defaults to the +load+ method
+  # instead if +thr <= 1+.
+  def self.load_parallel(file, thr, opts = {})
+    return self.load(file, opts) if thr <= 1
+
+    # Prepare data
+    lno = 0
+    File.open(file, 'r') { |fh| fh.each { lno += 1 } }
+    thr = lno if thr > lno
+    blk = (lno.to_f / thr).ceil
+
+    # Launch children processes
+    io  = []
+    pid = []
+    thr.times do |i|
+      io[i] = IO.pipe
+      pid << fork do
+        io[i][0].close
+        o = opts.merge(line_range: [i * blk, (i + 1) * blk - 1])
+        records = []
+        read_records(file, o) { |record| records << record }
+        Marshal.dump(records, io[i][1])
+        exit!(0)
+      end
+      io[i][1].close
+    end
+
+    # Collect and merge results
+    gfa = GFA.new(opts)
+    io.each_with_index do |pipe, k|
+      result = pipe[0].read
+      Process.wait(pid[k])
+      raise "Child process failed: #{k}" if result.empty?
+      Marshal.load(result).each { |record| gfa << record }
+      pipe[0].close
+    end
+
+    return gfa
+  end
+
   def self.supported_version?(v)
     v.to_f >= MIN_VERSION.to_f and v.to_f <= MAX_VERSION.to_f
   end
 
   # Instance-level
   def <<(obj)
-    obj = parse_line(obj) unless obj.is_a? GFA::Record
+    obj = GFA::Record[obj] unless obj.is_a? GFA::Record
     return if obj.nil? || obj.empty?
     @records[obj.type] << obj
 
-    if obj.type == :Header && !obj.fields[:VN].nil?
-      set_gfa_version(obj.fields[:VN].value)
+    if obj.type == :Header && !obj.VN.nil?
+      set_gfa_version(obj.VN.value)
     end
   end
 
   def set_gfa_version(v)
-    @gfa_version = v
-    unless GFA::supported_version? gfa_version
-      raise "GFA version currently unsupported: #{v}."
+    v = v.value if v.is_a? GFA::Field
+    unless GFA::supported_version? v
+      raise "GFA version currently unsupported: #{v}"
     end
+
+    @gfa_version = v
   end
-   
-  private
-      
-    def parse_line(ln)
-      ln.chomp!
-      return nil if ln =~ /^\s*$/
-      cols = ln.split("\t")
-      GFA::Record.code_class(cols.shift).new(*cols)
-    end
 end

data/lib/gfa/record/comment.rb

@@ -1,10 +1,15 @@
 class GFA::Record::Comment < GFA::Record
   CODE = :'#'
-  REQ_FIELDS = []
+  REQ_FIELDS = %i[comment]
   OPT_FIELDS = {}
+
+  REQ_FIELDS.each_index do |i|
+    define_method(REQ_FIELDS[i]) { fields[i + 2] }
+  end
    
-  def initialize(*opt_fields)
+  def initialize(comment, *opt_fields)
     @fields = {}
+    add_field(2, :Z, comment, /.*/)
     opt_fields.each { |f| add_opt_field(f, OPT_FIELDS) }
   end
 end

data/lib/gfa/record/containment.rb

@@ -1,3 +1,5 @@
+require 'gfa/record/has_from_to'
+
 class GFA::Record::Containment < GFA::Record
   CODE = :C
   REQ_FIELDS = %i[from from_orient to to_orient pos overlap]
@@ -10,20 +12,23 @@ class GFA::Record::Containment < GFA::Record
   REQ_FIELDS.each_index do |i|
     define_method(REQ_FIELDS[i]) { fields[i + 2] }
   end
+  OPT_FIELDS.each_key { |i| define_method(i) { fields[i] } }
+
+  include GFA::Record::HasFromTo
 
   alias container from
   alias container_orient from_orient
   alias contained to
   alias contained_orient to_orient
-   
+
   def initialize(from, from_orient, to, to_orient, pos, overlap, *opt_fields)
     @fields = {}
-    add_field(2, :Z, from, /^[!-)+-<>-~][!-~]*$/)
-    add_field(3, :Z, from_orient, /^+|-$/)
-    add_field(4, :Z, to, /^[!-)+-<>-~][!-~]*$/)
-    add_field(5, :Z, to_orient, /^+|-$/)
-    add_field(6, :i, pos, /^[0-9]*$/)
-    add_field(7, :Z, overlap, /^\*|([0-9]+[MIDNSHPX=])+$/)
+    add_field(2, :Z, from,        /[!-)+-<>-~][!-~]*/)
+    add_field(3, :Z, from_orient, /[+-]/)
+    add_field(4, :Z, to,          /[!-)+-<>-~][!-~]*/)
+    add_field(5, :Z, to_orient,   /[+-]/)
+    add_field(6, :i, pos,         /[0-9]*/)
+    add_field(7, :Z, overlap,     /\*|([0-9]+[MIDNSHPX=])+/)
     opt_fields.each { |f| add_opt_field(f, OPT_FIELDS) }
   end
 end

data/lib/gfa/record/has_from_to.rb

@@ -0,0 +1,47 @@
+module GFA::Record::HasFromTo
+  def from?(segment, orient = nil)
+    links_from_to?(segment, orient, true)
+  end
+
+  def to?(segment, orient = nil)
+    links_from_to?(segment, orient, false)
+  end
+
+  ##
+  # Extracts all linked segments from +gfa+ (which *must* be indexed)
+  def segments(gfa)
+    raise "Unindexed GFA" unless gfa.indexed?
+    [gfa.segments[from.value], gfa.segments[to.value]]
+  end
+
+  ##
+  # Include a GFA::Record::Segment +segment+?
+  def include?(segment)
+    # unless segment.is_a? GFA::Record::Segment
+    #   raise "Unrecognized class: #{segment.class}"
+    # end
+    segment.name == from || segment.name == to
+  end
+
+  ##
+  # Array of strings with the names of the segments linked by the
+  # record
+  def segment_names_a
+    [from.value, to.value]
+  end
+
+  private
+
+    def links_from_to?(segment, orient, from)
+      segment = segment_name(segment)
+      orient  = orient.value if orient.is_a? GFA::Field
+      base_k  = from ? 2 : 4
+      segment == fields[base_k].value &&
+        (orient.nil? || orient == fields[base_k + 1].value)
+    end
+
+    def segment_name(segment)
+      segment.is_a?(GFA::Record::Segment) ? segment.name.value :
+        segment.is_a?(GFA::Field) ? segment.value : segment
+    end
+end

data/lib/gfa/record/header.rb

@@ -4,6 +4,8 @@ class GFA::Record::Header < GFA::Record
   OPT_FIELDS = {
     VN: :Z # Version number
   }
+
+  OPT_FIELDS.each_key { |i| define_method(i) { fields[i] } }
    
   def initialize(*opt_fields)
     @fields = {}

data/lib/gfa/record/jump.rb

@@ -1,45 +1,26 @@
+require 'gfa/record/has_from_to'
+
 class GFA::Record::Jump < GFA::Record
   CODE = :J
   REQ_FIELDS = %i[from from_orient to to_orient distance]
   OPT_FIELDS = {
     SC: :i  # 1 indicates indirect shortcut connections. Only 0/1 allowed.
   }
-   
+
   REQ_FIELDS.each_index do |i|
     define_method(REQ_FIELDS[i]) { fields[i + 2] }
   end
+  OPT_FIELDS.each_key { |i| define_method(i) { fields[i] } }
+
+  include GFA::Record::HasFromTo
 
   def initialize(from, from_orient, to, to_orient, distance, *opt_fields)
     @fields = {}
-    add_field(2, :Z, from,        /^[!-)+-<>-~][!-~]*$/)
-    add_field(3, :Z, from_orient, /^+|-$/)
-    add_field(4, :Z, to,          /^[!-)+-<>-~][!-~]*$/)
-    add_field(5, :Z, to_orient,   /^+|-$/)
-    add_field(6, :Z, distance,    /^\*|[-+]?[0-9]+$/)
+    add_field(2, :Z, from,        /[!-)+-<>-~][!-~]*/)
+    add_field(3, :Z, from_orient, /[+-]/)
+    add_field(4, :Z, to,          /[!-)+-<>-~][!-~]*/)
+    add_field(5, :Z, to_orient,   /[+-]/)
+    add_field(6, :Z, distance,    /\*|[-+]?[0-9]+/)
     opt_fields.each { |f| add_opt_field(f, OPT_FIELDS) }
   end
-
-   
-  def from?(segment, orient = nil)
-    links_from_to?(segment, orient, true)
-  end
-   
-  def to?(segment, orient = nil)
-    links_from_to?(segment, orient, false)
-  end
-
-  private
-
-    def links_from_to?(segment, orient, from)
-      segment = segment_name(segment)
-      orient  = orient.value if orient.is_a? GFA::Field
-      base_k  = from ? 2 : 4
-      segment==fields[base_k].value &&
-        (orient.nil? || orient==fields[base_k + 1].value)
-    end
-
-    def segment_name(segment)
-      segment.is_a?(GFA::Record::Segment) ? segment.name.value :
-        segment.is_a?(GFA::Field) ? segment.value : segment
-    end
 end

data/lib/gfa/record/link.rb

@@ -1,3 +1,5 @@
+require 'gfa/record/has_from_to'
+
 class GFA::Record::Link < GFA::Record
   CODE = :L
   REQ_FIELDS = %i[from from_orient to to_orient overlap]
@@ -9,41 +11,21 @@ class GFA::Record::Link < GFA::Record
     KC: :i, # k-mer count
     ID: :Z  # Edge identifier
   }
-   
+
   REQ_FIELDS.each_index do |i|
     define_method(REQ_FIELDS[i]) { fields[i + 2] }
   end
+  OPT_FIELDS.each_key { |i| define_method(i) { fields[i] } }
+
+  include GFA::Record::HasFromTo
 
   def initialize(from, from_orient, to, to_orient, overlap, *opt_fields)
     @fields = {}
-    add_field(2, :Z, from, /^[!-)+-<>-~][!-~]*$/)
-    add_field(3, :Z, from_orient, /^+|-$/)
-    add_field(4, :Z, to, /^[!-)+-<>-~][!-~]*$/)
-    add_field(5, :Z, to_orient, /^+|-$/)
-    add_field(6, :Z, overlap, /^\*|([0-9]+[MIDNSHPX=])+$/)
+    add_field(2, :Z, from,        /[!-)+-<>-~][!-~]*/)
+    add_field(3, :Z, from_orient, /[+-]/)
+    add_field(4, :Z, to,          /[!-)+-<>-~][!-~]*/)
+    add_field(5, :Z, to_orient,   /[+-]/)
+    add_field(6, :Z, overlap,     /\*|([0-9]+[MIDNSHPX=])+/)
     opt_fields.each { |f| add_opt_field(f, OPT_FIELDS) }
   end
-
-  def from?(segment, orient = nil)
-    links_from_to?(segment, orient, true)
-  end
-   
-  def to?(segment, orient = nil)
-    links_from_to?(segment, orient, false)
-  end
-
-  private
-
-    def links_from_to?(segment, orient, from)
-      segment = segment_name(segment)
-      orient  = orient.value if orient.is_a? GFA::Field
-      base_k  = from ? 2 : 4
-      segment==fields[base_k].value &&
-        (orient.nil? || orient==fields[base_k + 1].value)
-    end
-
-    def segment_name(segment)
-      segment.is_a?(GFA::Record::Segment) ? segment.name.value :
-        segment.is_a?(GFA::Field) ? segment.value : segment
-    end
 end

data/lib/gfa/record/path.rb

@@ -1,19 +1,45 @@
 class GFA::Record::Path < GFA::Record
   CODE = :P
-  REQ_FIELDS = %i[path_name segment_name cigar]
+  REQ_FIELDS = %i[path_name segment_names overlaps]
   OPT_FIELDS = {}
 
   REQ_FIELDS.each_index do |i|
     define_method(REQ_FIELDS[i]) { fields[i + 2] }
   end
 
-  alias overlaps cigar
+  alias segment_name segment_names
+  alias cigar overlaps
 
-  def initialize(path_name, segment_name, cigar, *opt_fields)
+  def initialize(path_name, segment_names, overlaps, *opt_fields)
     @fields = {}
-    add_field(2, :Z, path_name, /^[!-)+-<>-~][!-~]*$/)
-    add_field(3, :Z, segment_name, /^[!-)+-<>-~][!-~]*$/)
-    add_field(4, :Z, cigar, /^\*|([0-9]+[MIDNSHPX=])+$/)
+    add_field(2, :Z, path_name,     /[!-)+-<>-~][!-~]*/)
+    add_field(3, :Z, segment_names, /[!-)+-<>-~][!-~]*/)
+    add_field(4, :Z, overlaps,      /\*|([0-9]+[MIDNSHPX=]|[-+]?[0-9]+J|.)+/)
     opt_fields.each { |f| add_opt_field(f, OPT_FIELDS) }
   end
+
+  ##
+  # Array of segment names (without orientations) as strings
+  def segment_names_a
+    segment_names.value.split(/[,;]/).map { |i| i.gsub(/[+-]$/, '') }
+  end
+
+  ##
+  # Extracts all linked segments from +gfa+ (which *must* be indexed)
+  def segments(gfa)
+    raise "Unindexed GFA" unless gfa.indexed?
+    segment_names_a.map do |name|
+      gfa.segments[name]
+    end
+  end
+
+  ##
+  # Includes a GFA::Record::Segment +segment+?
+  def include?(segment)
+    # unless segment.is_a? GFA::Record::Segment
+    #   raise "Unrecognized class: #{segment.class}"
+    # end
+
+    segment_names_a.any? { |name| segment.name == name }
+  end
 end

data/lib/gfa/record/segment.rb

@@ -8,18 +8,22 @@ class GFA::Record::Segment < GFA::Record
     KC: :i, # k-mer count
     SH: :H, # SHA-256 checksum of the sequence
     UR: :Z, # URI or local file-system path of the sequence
-    # Non-cannonical
-    DP: :f  # (From SAM)
+    # Non-cannonical but uppercase (thus, reserved)
+    DP: :f, # SAM
+    SN: :Z, # rGFA: Name of stable sequence from which the segment is derived
+    SO: :i, # rGFA: Offset on the stable sequence
+    SR: :i  # rGFA: Rank. 0 if on a linear reference genome; >0 otherwise
   }
 
   REQ_FIELDS.each_index do |i|
     define_method(REQ_FIELDS[i]) { fields[i + 2] }
   end
+  OPT_FIELDS.each_key { |i| define_method(i) { fields[i] } }
    
   def initialize(name, sequence, *opt_fields)
     @fields = {}
-    add_field(2, :Z, name, /^[!-)+-<>-~][!-~]*$/)
-    add_field(3, :Z, sequence, /^\*|[A-Za-z=.]+$/)
+    add_field(2, :Z, name,     /[!-)+-<>-~][!-~]*/)
+    add_field(3, :Z, sequence, /\*|[A-Za-z=.]+/)
     opt_fields.each { |f| add_opt_field(f, OPT_FIELDS) }
   end
 end

data/lib/gfa/record/walk.rb

@@ -9,12 +9,12 @@ class GFA::Record::Walk < GFA::Record
 
   def initialize(sample_id, hap_index, seq_id, seq_start, seq_end, walk, *opt_fields)
     @fields = {}
-    add_field(2, :Z, sample_id, /^[!-)+-<>-~][!-~]*$/)
-    add_field(3, :i, hap_index, /^[0-9]+$/)
-    add_field(4, :Z, seq_id, /^[!-)+-<>-~][!-~]*$/)
-    add_field(5, :i, seq_start, /^\*|[0-9]+$/)
-    add_field(6, :i, seq_end, /^\*|[0-9]+$/)
-    add_field(7, :Z, walk, /^([><][!-;=?-~]+)+$/)
+    add_field(2, :Z, sample_id, /[!-)+-<>-~][!-~]*/)
+    add_field(3, :i, hap_index, /[0-9]+/)
+    add_field(4, :Z, seq_id,    /[!-)+-<>-~][!-~]*/)
+    add_field(5, :i, seq_start, /\*|[0-9]+/)
+    add_field(6, :i, seq_end,   /\*|[0-9]+/)
+    add_field(7, :Z, walk,      /([><][!-;=?-~]+)+/)
     opt_fields.each { |f| add_opt_field(f, OPT_FIELDS) }
   end
 end