htslib 0.3.2 → 0.4.0

data/lib/hts/bcf/record.rb

@@ -54,11 +54,11 @@ module HTS
       end
 
       def id=(id)
-        LibHTS.bcf_update_id(@header, @bcf1, id)
+        LibHTS.bcf_update_id(@header.struct, @bcf1, id)
       end
 
       def clear_id
-        LibHTS.bcf_update_id(@header, @bcf1, ".")
+        LibHTS.bcf_update_id(@header.struct, @bcf1, ".")
       end
 
       def ref
@@ -100,7 +100,7 @@ module HTS
         when 1
           id = d[:flt].read_int
           LibHTS.bcf_hdr_int2id(@header.struct, LibHTS::BCF_DT_ID, id)
-        when 2..nil
+        when 2..
           d[:flt].get_array_of_int(0, n_flt).map do |i|
             LibHTS.bcf_hdr_int2id(@header.struct, LibHTS::BCF_DT_ID, i)
           end
@@ -130,9 +130,13 @@ module HTS
 
       def to_s
         ksr = LibHTS::KString.new
-        raise "Failed to format record" if LibHTS.vcf_format(@header.struct, @bcf1, ksr) == -1
+        begin
+          raise "Failed to format record" if LibHTS.vcf_format(@header.struct, @bcf1, ksr) == -1
 
-        ksr[:s]
+          ksr.read_string_copy
+        ensure
+          ksr.free_buffer
+        end
       end
 
       private

data/lib/hts/bcf.rb

@@ -3,6 +3,7 @@
 require_relative "../htslib"
 
 require_relative "hts"
+require_relative "bcf/errors"
 require_relative "bcf/header"
 require_relative "bcf/info"
 require_relative "bcf/format"
@@ -27,8 +28,27 @@ module HTS
       file
     end
 
+    def self.build_index(file_name, index_name = nil, min_shift = 14, threads = 0, verbose = true)
+      if verbose
+        if index_name
+          warn "Create index for #{file_name} to #{index_name}"
+        else
+          warn "Create index for #{file_name}"
+        end
+      end
+
+      case LibHTS.bcf_index_build3(file_name, index_name, min_shift, threads)
+      when 0 # successful
+      when -1 then raise IndexError, "Indexing failed for #{file_name}"
+      when -2 then raise IndexError, "Opening #{file_name} failed while building the index"
+      when -3 then raise IndexError, "#{file_name} is not in an indexable format"
+      when -4 then raise IndexError, "Failed to create or save the index for #{file_name}"
+      else raise IndexError, "Unknown index build error for #{file_name}"
+      end
+    end
+
     def initialize(file_name, mode = "r", index: nil, threads: nil,
-                   build_index: false)
+                   build_index: false, subset: nil)
       if block_given?
         message = "HTS::Bcf.new() does not take block; Please use HTS::Bcf.open() instead"
         raise message
@@ -42,43 +62,43 @@ module HTS
       @nthreads   = threads
       @hts_file   = LibHTS.hts_open(@file_name, mode)
 
-      raise Errno::ENOENT, "Failed to open #{@file_name}" if @hts_file.null?
+      raise OpenError, "Failed to open #{@file_name}" if @hts_file.null?
 
       set_threads(threads) if threads
 
+      raise SubsetError, "Sample subsetting is only available when reading BCF/VCF files" if subset && @mode[0] == "w"
+
       return if @mode[0] == "w"
 
-      @header = Bcf::Header.new(@hts_file)
+      @read_header = Bcf::Header.new(@hts_file)
+      @header = subset ? @read_header.subset(subset) : @read_header
       build_index(index) if build_index
       @idx = load_index(index)
       @start_position = tell
     end
 
-    def build_index(index_name = nil, min_shift: 14, threads: 2)
+    def build_index(index_name = nil, min_shift: 14, verbose: true)
       check_closed
 
-      if index_name
-        warn "Create index for #{@file_name} to #{index_name}"
-      else
-        warn "Create index for #{@file_name}"
-      end
-      case LibHTS.bcf_index_build3(@file_name, index_name, min_shift, @nthreads || threads)
-      when 0 # successful
-      when -1 then raise "indexing failed"
-      when -2 then raise "opening #{@file_name} failed"
-      when -3 then raise "format not indexable"
-      when -4 then raise "failed to create and/or save the index"
-      else raise "unknown error"
-      end
+      self.class.build_index(@file_name, index_name, min_shift, @nthreads || 0, verbose)
       self # for method chaining
     end
 
     def load_index(index_name = nil)
       check_closed
 
-      if index_name
+      if file_format == "vcf"
+        @index_format = :tabix
+        if index_name
+          LibHTS.tbx_index_load2(@file_name, index_name)
+        else
+          LibHTS.tbx_index_load3(@file_name, nil, 2)
+        end
+      elsif index_name
+        @index_format = :bcf
         LibHTS.bcf_index_load2(@file_name, index_name)
       else
+        @index_format = :bcf
         LibHTS.bcf_index_load3(@file_name, nil, 2)
       end
     end
@@ -90,7 +110,14 @@ module HTS
     end
 
     def close
-      LibHTS.hts_idx_destroy(@idx) if @idx && !@idx.null?
+      if @idx && !@idx.null?
+        case @index_format
+        when :bcf
+          LibHTS.hts_idx_destroy(@idx)
+        when :tabix
+          @idx.close
+        end
+      end
       @idx = nil
       super
     end
@@ -98,7 +125,7 @@ module HTS
     def write_header(header)
       check_closed
 
-      @header = header.clone
+      @header = header.dup
       LibHTS.bcf_hdr_write(@hts_file, header)
     end
 
@@ -212,8 +239,7 @@ module HTS
     def query(region, beg = nil, end_ = nil, copy: false, &block)
       check_closed
 
-      raise "query is only available for BCF files" unless file_format == "bcf"
-      raise "Index file is required to call the query method." unless index_loaded?
+      raise MissingIndexError, "Index file is required to call the query method for #{@file_name}" unless index_loaded?
 
       case region
       when Array
@@ -261,8 +287,10 @@ module HTS
     def queryi_reuse(tid, beg, end_, &block)
       return to_enum(__method__, tid, beg, end_) unless block_given?
 
+      return queryi_reuse_vcf(tid, beg, end_, &block) if tabix_index?
+
       qiter = LibHTS.bcf_itr_queryi(@idx, tid, beg, end_)
-      raise "Failed to query region #{tid} #{beg} #{end_}" if qiter.null?
+      raise QueryError, "Failed to query region #{tid}:#{beg}-#{end_} in #{@file_name}" if qiter.null?
 
       query_reuse_yield(qiter, &block)
       self
@@ -271,8 +299,10 @@ module HTS
     def querys_reuse(region, &block)
       return to_enum(__method__, region) unless block_given?
 
-      qiter = LibHTS.bcf_itr_querys(@idx, header, region)
-      raise "Failed to query region #{region}" if qiter.null?
+      return querys_reuse_vcf(region, &block) if tabix_index?
+
+      qiter = LibHTS.bcf_itr_querys(@idx, read_header, region)
+      raise QueryError, "Failed to query region #{region.inspect} in #{@file_name}" if qiter.null?
 
       query_reuse_yield(qiter, &block)
       self
@@ -296,6 +326,7 @@ module HTS
           break if slen == -1
           raise if slen < -1
 
+          apply_subset!(record)
           yield record
         end
       ensure
@@ -306,8 +337,10 @@ module HTS
     def queryi_copy(tid, beg, end_, &block)
       return to_enum(__method__, tid, beg, end_) unless block_given?
 
+      return queryi_copy_vcf(tid, beg, end_, &block) if tabix_index?
+
       qiter = LibHTS.bcf_itr_queryi(@idx, tid, beg, end_)
-      raise "Failed to query region #{tid} #{beg} #{end_}" if qiter.null?
+      raise QueryError, "Failed to query region #{tid}:#{beg}-#{end_} in #{@file_name}" if qiter.null?
 
       query_copy_yield(qiter, &block)
       self
@@ -316,8 +349,10 @@ module HTS
     def querys_copy(region, &block)
       return to_enum(__method__, region) unless block_given?
 
-      qiter = LibHTS.bcf_itr_querys(@idx, header, region)
-      raise "Failed to query region #{region}" if qiter.null?
+      return querys_copy_vcf(region, &block) if tabix_index?
+
+      qiter = LibHTS.bcf_itr_querys(@idx, read_header, region)
+      raise QueryError, "Failed to query region #{region.inspect} in #{@file_name}" if qiter.null?
 
       query_copy_yield(qiter, &block)
       self
@@ -333,18 +368,94 @@ module HTS
     end
 
     def query_copy_yield(qiter)
+      bcf1 = LibHTS.bcf_init
+      record = Record.new(header, bcf1)
       loop do
-        bcf1 = LibHTS.bcf_init
         slen = LibHTS.hts_itr_next(@hts_file[:fp][:bgzf], qiter, bcf1, ::FFI::Pointer::NULL)
         break if slen == -1
         raise if slen < -1
 
-        yield Record.new(header, bcf1)
+        apply_subset!(record)
+        yield record.dup
       end
     ensure
       LibHTS.bcf_itr_destroy(qiter)
     end
 
+    def tabix_index?
+      @index_format == :tabix
+    end
+
+    def queryi_reuse_vcf(tid, beg, end_, &block)
+      qiter = LibHTS.tbx_itr_queryi(@idx, tid, beg, end_)
+      raise QueryError, "Failed to query region #{tid}:#{beg}-#{end_} in #{@file_name}" if qiter.null?
+
+      query_reuse_yield_vcf(qiter, &block)
+      self
+    end
+
+    def querys_reuse_vcf(region, &block)
+      qiter = LibHTS.tbx_itr_querys(@idx, region)
+      raise QueryError, "Failed to query region #{region.inspect} in #{@file_name}" if qiter.null?
+
+      query_reuse_yield_vcf(qiter, &block)
+      self
+    end
+
+    def query_reuse_yield_vcf(qiter)
+      line = LibHTS::KString.new
+      bcf1 = LibHTS.bcf_init
+      record = Record.new(header, bcf1)
+      begin
+        while (slen = LibHTS.tbx_itr_next(@hts_file, @idx, qiter, line)) >= 0
+          raise QueryError, "Failed to parse VCF record in #{@file_name}" if LibHTS.vcf_parse(line, read_header,
+                                                                                              bcf1) < 0
+
+          apply_subset!(record)
+          yield record
+        end
+        raise if slen < -1
+      ensure
+        line.free_buffer
+        LibHTS.hts_itr_destroy(qiter)
+      end
+    end
+
+    def queryi_copy_vcf(tid, beg, end_, &block)
+      qiter = LibHTS.tbx_itr_queryi(@idx, tid, beg, end_)
+      raise QueryError, "Failed to query region #{tid}:#{beg}-#{end_} in #{@file_name}" if qiter.null?
+
+      query_copy_yield_vcf(qiter, &block)
+      self
+    end
+
+    def querys_copy_vcf(region, &block)
+      qiter = LibHTS.tbx_itr_querys(@idx, region)
+      raise QueryError, "Failed to query region #{region.inspect} in #{@file_name}" if qiter.null?
+
+      query_copy_yield_vcf(qiter, &block)
+      self
+    end
+
+    def query_copy_yield_vcf(qiter)
+      line = LibHTS::KString.new
+      begin
+        while (slen = LibHTS.tbx_itr_next(@hts_file, @idx, qiter, line)) >= 0
+          bcf1 = LibHTS.bcf_init
+          raise QueryError, "Failed to parse VCF record in #{@file_name}" if LibHTS.vcf_parse(line, read_header,
+                                                                                              bcf1) < 0
+
+          record = Record.new(header, bcf1)
+          apply_subset!(record)
+          yield record
+        end
+        raise if slen < -1
+      ensure
+        line.free_buffer
+        LibHTS.hts_itr_destroy(qiter)
+      end
+    end
+
     def each_record_reuse
       check_closed
 
@@ -352,7 +463,10 @@ module HTS
 
       bcf1 = LibHTS.bcf_init
       record = Record.new(header, bcf1)
-      yield record while LibHTS.bcf_read(@hts_file, header, bcf1) != -1
+      while LibHTS.bcf_read(@hts_file, read_header, bcf1) != -1
+        apply_subset!(record)
+        yield record
+      end
       self
     end
 
@@ -361,11 +475,26 @@ module HTS
 
       return to_enum(__method__) unless block_given?
 
-      while LibHTS.bcf_read(@hts_file, header, bcf1 = LibHTS.bcf_init) != -1
-        record = Record.new(header, bcf1)
-        yield record
+      bcf1 = LibHTS.bcf_init
+      record = Record.new(header, bcf1)
+      while LibHTS.bcf_read(@hts_file, read_header, bcf1) != -1
+        apply_subset!(record)
+        yield record.dup
       end
       self
     end
+
+    def read_header
+      @read_header || header
+    end
+
+    def apply_subset!(record)
+      return unless header.subset?
+
+      rc = LibHTS.bcf_subset(header.struct, record.struct, header.subset_sample_count, header.subset_imap_pointer || ::FFI::Pointer::NULL)
+      return if rc >= 0
+
+      raise SubsetError, "Failed to subset samples #{header.subset_samples.inspect} while reading #{@file_name}"
+    end
   end
 end

data/lib/hts/faidx.rb

@@ -1,16 +1,21 @@
 # frozen_string_literal: true
 
 require_relative "../htslib"
-require_relative "faidx/sequence"
 
 module HTS
-  class Faidx
-    include Enumerable
+  module LibC
+    extend FFI::Library
+    ffi_lib FFI::Library::LIBC
+    attach_function :free, [:pointer], :void
+  end
+end
 
-    attr_reader :file_name
+module HTS
+  class Faidx
+    attr_reader :file_name, :format
 
-    def self.open(*args, **kw)
-      file = new(*args, **kw) # do not yield
+    def self.open(file_name, format: :auto, auto_build: true)
+      file = new(file_name, format:, auto_build:) # do not yield
       return file unless block_given?
 
       begin
@@ -21,16 +26,19 @@ module HTS
       file
     end
 
-    def initialize(file_name)
+    def self.build_index(file_name, fai_path = nil, gzi_path = nil)
+      case LibHTS.fai_build3(file_name, fai_path, gzi_path)
+      when 0
+      else raise HTS::Error, "Failed to build faidx index for #{file_name}"
+      end
+    end
+
+    def initialize(file_name, format: :auto, auto_build: true)
       raise ArgumentError, "HTS::Faidx.new() does not take block; Please use HTS::Faidx.open() instead" if block_given?
 
       @file_name = file_name.freeze
-      @fai = case File.extname(@file_name)
-             when ".fq", ".fastq"
-               LibHTS.fai_load_format(@file_name, 2)
-             else
-               LibHTS.fai_load(@file_name)
-             end
+      @format = resolve_format(@file_name, format)
+      @fai = load_handle(@file_name, @format, auto_build)
 
       raise Errno::ENOENT, "Failed to open #{@file_name}" if @fai.null?
     end
@@ -50,42 +58,19 @@ module HTS
       @fai.nil? || @fai.null?
     end
 
-    def file_format
-      check_closed
-      @fai[:format]
-    end
-
-    # Iterate over each sequence in the index.
-    # @yield [Sequence] each sequence object
-    # @return [Enumerator] if no block given
-    def each
-      return to_enum(__method__) unless block_given?
-
-      check_closed
-      names.each { |name| yield self[name] }
-    end
-
-    # the number of sequences in the index.
-    # @return [Integer] the number of sequences
-    def length
+    def size
       check_closed
       LibHTS.faidx_nseq(@fai)
     end
-    alias size length
 
-    # Return the list of sequence names in the index.
-    # @return [Array<String>] sequence names
+    alias length size
+
     def names
       check_closed
       Array.new(length) { |i| LibHTS.faidx_iseq(@fai, i) }
     end
 
-    alias keys names
-
-    # Check if a sequence exists in the index.
-    # @param key [String, Symbol] sequence name
-    # @return [Boolean] true if the sequence exists
-    def has_key?(key)
+    def has_seq?(key)
       check_closed
       raise ArgumentError, "Expect chrom to be String or Symbol" unless key.is_a?(String) || key.is_a?(Symbol)
 
@@ -97,94 +82,57 @@ module HTS
       end
     end
 
-    # Get a Sequence object by name or index.
-    # @param name [String, Symbol, Integer] sequence name or index
-    # @return [Sequence] the sequence object
-    # @raise [ArgumentError] if the sequence does not exist
-    def [](name)
-      check_closed
-      name = LibHTS.faidx_iseq(@fai, name) if name.is_a?(Integer)
-      Sequence.new(self, name)
-    end
-
-    # Return the length of the requested chromosome.
-    # @param chrom [String, Symbol] chromosome name
-    # @return [Integer] sequence length
-    # @raise [ArgumentError] if the sequence does not exist
     def seq_len(chrom)
       check_closed
       raise ArgumentError, "Expect chrom to be String or Symbol" unless chrom.is_a?(String) || chrom.is_a?(Symbol)
 
       chrom = chrom.to_s
-      result = LibHTS.faidx_seq_len(@fai, chrom)
+      result = LibHTS.faidx_seq_len64(@fai, chrom)
       raise ArgumentError, "Sequence not found: #{chrom}" if result == -1
 
       result
     end
 
-    # @overload fetch_seq(name)
-    #   Fetch the sequence as a String.
-    #   @param name [String, Symbol] chr1:0-10
-    #   @return [String] the sequence
-    # @overload fetch_seq(name, start, stop)
-    #   Fetch the sequence as a String.
-    #   @param name [String, Symbol] the name of the chromosome
-    #   @param start [Integer] the start position of the sequence (0-based)
-    #   @param stop [Integer] the end position of the sequence (0-based)
-    #   @return [String] the sequence
     def fetch_seq(name, start = nil, stop = nil)
       check_closed
       name = name.to_s
-      rlen = FFI::MemoryPointer.new(:int)
 
       if start.nil? && stop.nil?
-        result = LibHTS.fai_fetch64(@fai, name, rlen)
+        len = seq_len(name)
+        return "" if len.zero?
+
+        fetch_seq(name, 0, len - 1)
       else
         validate_range!(name, start, stop)
+        rlen = FFI::MemoryPointer.new(:int64)
         result = LibHTS.faidx_fetch_seq64(@fai, name, start, stop, rlen)
+        fetch_result(result, rlen.read_int64, "sequence", name, start, stop)
       end
-
-      case rlen.read_int
-      when -2 then raise ArgumentError, "Invalid chromosome name: #{name}"
-      when -1 then raise HTS::Error, "Error fetching sequence: #{name}:#{start}-#{stop}"
-      end
-
-      result
     end
 
-    alias seq fetch_seq
-
-    # @overload fetch_qual(name)
-    #   Fetch the quality string.
-    #   @param name [String, Symbol] sequence name
-    #   @return [String] the quality string
-    # @overload fetch_qual(name, start, stop)
-    #   Fetch the quality string.
-    #   @param name [String, Symbol] the name of the chromosome
-    #   @param start [Integer] the start position of the sequence (0-based)
-    #   @param stop [Integer] the end position of the sequence (0-based)
-    #   @return [String] the quality string
     def fetch_qual(name, start = nil, stop = nil)
       check_closed
+      raise HTS::Error, "Quality is only available for FASTQ indexes" unless format == :fastq
+
       name = name.to_s
-      rlen = FFI::MemoryPointer.new(:int)
 
       if start.nil? && stop.nil?
-        result = LibHTS.fai_fetchqual64(@fai, name, rlen)
+        len = seq_len(name)
+        return "" if len.zero?
+
+        fetch_qual(name, 0, len - 1)
       else
         validate_range!(name, start, stop)
+        rlen = FFI::MemoryPointer.new(:int64)
         result = LibHTS.faidx_fetch_qual64(@fai, name, start, stop, rlen)
+        fetch_result(result, rlen.read_int64, "quality", name, start, stop)
       end
-
-      case rlen.read_int
-      when -2 then raise ArgumentError, "Invalid chromosome name: #{name}"
-      when -1 then raise HTS::Error, "Error fetching quality: #{name}:#{start}-#{stop}"
-      end
-
-      result
     end
 
-    alias qual fetch_qual
+    def build_index(fai_path = nil, gzi_path = nil)
+      self.class.build_index(@file_name, fai_path, gzi_path)
+      self
+    end
 
     private
 
@@ -192,19 +140,54 @@ module HTS
       raise IOError, "closed Faidx" if closed?
     end
 
-    # Validate range parameters.
-    # @param name [String] sequence name
-    # @param start [Integer] start position (0-based)
-    # @param stop [Integer] stop position (0-based)
-    # @raise [ArgumentError] if range is invalid
     def validate_range!(name, start, stop)
       raise ArgumentError, "Expect start to be >= 0" if start < 0
       raise ArgumentError, "Expect stop to be >= 0" if stop < 0
       raise ArgumentError, "Expect start to be <= stop" if start > stop
 
       len = seq_len(name)
-      raise ArgumentError, "Sequence not found: #{name}" if len.nil?
       raise ArgumentError, "Expect stop to be < seq_len (#{len})" if stop >= len
     end
+
+    def fetch_result(ptr, len, kind, name, start, stop)
+      case len
+      when -2 then raise ArgumentError, "Sequence not found: #{name}"
+      when -1 then raise HTS::Error, "Error fetching #{kind}: #{name}:#{start}-#{stop}"
+      end
+
+      raise HTS::Error, "Error fetching #{kind}: #{name}:#{start}-#{stop}" if ptr.null?
+
+      begin
+        ptr.read_string_length(len)
+      ensure
+        HTS::LibC.free(ptr)
+      end
+    end
+
+    def load_handle(file_name, format, auto_build)
+      case [format, auto_build]
+      when [:fasta, true]
+        LibHTS.fai_load_format(file_name, :FAI_FASTA)
+      when [:fastq, true]
+        LibHTS.fai_load_format(file_name, :FAI_FASTQ)
+      when [:fasta, false]
+        LibHTS.fai_load3_format(file_name, nil, nil, 0, :FAI_FASTA)
+      when [:fastq, false]
+        LibHTS.fai_load3_format(file_name, nil, nil, 0, :FAI_FASTQ)
+      else
+        raise ArgumentError, "Unsupported format: #{format}"
+      end
+    end
+
+    def resolve_format(file_name, format)
+      case format
+      when :auto
+        file_name.match?(/\.(fastq|fq)(\.gz|\.bgz)?\z/i) ? :fastq : :fasta
+      when :fasta, :fastq
+        format
+      else
+        raise ArgumentError, "Unsupported format: #{format}"
+      end
+    end
   end
 end

data/lib/hts/hts.rb

@@ -71,7 +71,10 @@ module HTS
 
     def fai=(fai)
       check_closed
-      LibHTS.hts_set_fai_filename(@hts_file, fai) > 0 || raise
+      r = LibHTS.hts_set_fai_filename(@hts_file, fai)
+      raise "Failed to load fasta index: #{fai}" if r.negative?
+
+      self
     end
 
     def set_threads(n = nil)