htslib 0.3.1 → 0.4.0

data/lib/hts/bcf.rb

@@ -3,6 +3,7 @@
 require_relative "../htslib"
 
 require_relative "hts"
+require_relative "bcf/errors"
 require_relative "bcf/header"
 require_relative "bcf/info"
 require_relative "bcf/format"
@@ -27,8 +28,27 @@ module HTS
       file
     end
 
+    def self.build_index(file_name, index_name = nil, min_shift = 14, threads = 0, verbose = true)
+      if verbose
+        if index_name
+          warn "Create index for #{file_name} to #{index_name}"
+        else
+          warn "Create index for #{file_name}"
+        end
+      end
+
+      case LibHTS.bcf_index_build3(file_name, index_name, min_shift, threads)
+      when 0 # successful
+      when -1 then raise IndexError, "Indexing failed for #{file_name}"
+      when -2 then raise IndexError, "Opening #{file_name} failed while building the index"
+      when -3 then raise IndexError, "#{file_name} is not in an indexable format"
+      when -4 then raise IndexError, "Failed to create or save the index for #{file_name}"
+      else raise IndexError, "Unknown index build error for #{file_name}"
+      end
+    end
+
     def initialize(file_name, mode = "r", index: nil, threads: nil,
-                   build_index: false)
+                   build_index: false, subset: nil)
       if block_given?
         message = "HTS::Bcf.new() does not take block; Please use HTS::Bcf.open() instead"
         raise message
@@ -42,43 +62,43 @@ module HTS
       @nthreads   = threads
       @hts_file   = LibHTS.hts_open(@file_name, mode)
 
-      raise Errno::ENOENT, "Failed to open #{@file_name}" if @hts_file.null?
+      raise OpenError, "Failed to open #{@file_name}" if @hts_file.null?
 
       set_threads(threads) if threads
 
+      raise SubsetError, "Sample subsetting is only available when reading BCF/VCF files" if subset && @mode[0] == "w"
+
       return if @mode[0] == "w"
 
-      @header = Bcf::Header.new(@hts_file)
+      @read_header = Bcf::Header.new(@hts_file)
+      @header = subset ? @read_header.subset(subset) : @read_header
       build_index(index) if build_index
       @idx = load_index(index)
       @start_position = tell
     end
 
-    def build_index(index_name = nil, min_shift: 14, threads: 2)
+    def build_index(index_name = nil, min_shift: 14, verbose: true)
       check_closed
 
-      if index_name
-        warn "Create index for #{@file_name} to #{index_name}"
-      else
-        warn "Create index for #{@file_name}"
-      end
-      case LibHTS.bcf_index_build3(@file_name, index_name, min_shift, @nthreads || threads)
-      when 0 # successful
-      when -1 then raise "indexing failed"
-      when -2 then raise "opening #{@file_name} failed"
-      when -3 then raise "format not indexable"
-      when -4 then raise "failed to create and/or save the index"
-      else raise "unknown error"
-      end
+      self.class.build_index(@file_name, index_name, min_shift, @nthreads || 0, verbose)
       self # for method chaining
     end
 
     def load_index(index_name = nil)
       check_closed
 
-      if index_name
+      if file_format == "vcf"
+        @index_format = :tabix
+        if index_name
+          LibHTS.tbx_index_load2(@file_name, index_name)
+        else
+          LibHTS.tbx_index_load3(@file_name, nil, 2)
+        end
+      elsif index_name
+        @index_format = :bcf
         LibHTS.bcf_index_load2(@file_name, index_name)
       else
+        @index_format = :bcf
         LibHTS.bcf_index_load3(@file_name, nil, 2)
       end
     end
@@ -90,7 +110,14 @@ module HTS
     end
 
     def close
-      LibHTS.hts_idx_destroy(@idx) if @idx && !@idx.null?
+      if @idx && !@idx.null?
+        case @index_format
+        when :bcf
+          LibHTS.hts_idx_destroy(@idx)
+        when :tabix
+          @idx.close
+        end
+      end
       @idx = nil
       super
     end
@@ -98,7 +125,7 @@ module HTS
     def write_header(header)
       check_closed
 
-      @header = header.clone
+      @header = header.dup
       LibHTS.bcf_hdr_write(@hts_file, header)
     end
 
@@ -212,8 +239,7 @@ module HTS
     def query(region, beg = nil, end_ = nil, copy: false, &block)
       check_closed
 
-      raise "query is only available for BCF files" unless file_format == "bcf"
-      raise "Index file is required to call the query method." unless index_loaded?
+      raise MissingIndexError, "Index file is required to call the query method for #{@file_name}" unless index_loaded?
 
       case region
       when Array
@@ -261,8 +287,10 @@ module HTS
     def queryi_reuse(tid, beg, end_, &block)
       return to_enum(__method__, tid, beg, end_) unless block_given?
 
+      return queryi_reuse_vcf(tid, beg, end_, &block) if tabix_index?
+
       qiter = LibHTS.bcf_itr_queryi(@idx, tid, beg, end_)
-      raise "Failed to query region #{tid} #{beg} #{end_}" if qiter.null?
+      raise QueryError, "Failed to query region #{tid}:#{beg}-#{end_} in #{@file_name}" if qiter.null?
 
       query_reuse_yield(qiter, &block)
       self
@@ -271,8 +299,10 @@ module HTS
     def querys_reuse(region, &block)
       return to_enum(__method__, region) unless block_given?
 
-      qiter = LibHTS.bcf_itr_querys(@idx, header, region)
-      raise "Failed to query region #{region}" if qiter.null?
+      return querys_reuse_vcf(region, &block) if tabix_index?
+
+      qiter = LibHTS.bcf_itr_querys(@idx, read_header, region)
+      raise QueryError, "Failed to query region #{region.inspect} in #{@file_name}" if qiter.null?
 
       query_reuse_yield(qiter, &block)
       self
@@ -296,6 +326,7 @@ module HTS
           break if slen == -1
           raise if slen < -1
 
+          apply_subset!(record)
           yield record
         end
       ensure
@@ -306,8 +337,10 @@ module HTS
     def queryi_copy(tid, beg, end_, &block)
       return to_enum(__method__, tid, beg, end_) unless block_given?
 
+      return queryi_copy_vcf(tid, beg, end_, &block) if tabix_index?
+
       qiter = LibHTS.bcf_itr_queryi(@idx, tid, beg, end_)
-      raise "Failed to query region #{tid} #{beg} #{end_}" if qiter.null?
+      raise QueryError, "Failed to query region #{tid}:#{beg}-#{end_} in #{@file_name}" if qiter.null?
 
       query_copy_yield(qiter, &block)
       self
@@ -316,8 +349,10 @@ module HTS
     def querys_copy(region, &block)
       return to_enum(__method__, region) unless block_given?
 
-      qiter = LibHTS.bcf_itr_querys(@idx, header, region)
-      raise "Failed to query region #{region}" if qiter.null?
+      return querys_copy_vcf(region, &block) if tabix_index?
+
+      qiter = LibHTS.bcf_itr_querys(@idx, read_header, region)
+      raise QueryError, "Failed to query region #{region.inspect} in #{@file_name}" if qiter.null?
 
       query_copy_yield(qiter, &block)
       self
@@ -333,18 +368,94 @@ module HTS
     end
 
     def query_copy_yield(qiter)
+      bcf1 = LibHTS.bcf_init
+      record = Record.new(header, bcf1)
       loop do
-        bcf1 = LibHTS.bcf_init
         slen = LibHTS.hts_itr_next(@hts_file[:fp][:bgzf], qiter, bcf1, ::FFI::Pointer::NULL)
         break if slen == -1
         raise if slen < -1
 
-        yield Record.new(header, bcf1)
+        apply_subset!(record)
+        yield record.dup
       end
     ensure
       LibHTS.bcf_itr_destroy(qiter)
     end
 
+    def tabix_index?
+      @index_format == :tabix
+    end
+
+    def queryi_reuse_vcf(tid, beg, end_, &block)
+      qiter = LibHTS.tbx_itr_queryi(@idx, tid, beg, end_)
+      raise QueryError, "Failed to query region #{tid}:#{beg}-#{end_} in #{@file_name}" if qiter.null?
+
+      query_reuse_yield_vcf(qiter, &block)
+      self
+    end
+
+    def querys_reuse_vcf(region, &block)
+      qiter = LibHTS.tbx_itr_querys(@idx, region)
+      raise QueryError, "Failed to query region #{region.inspect} in #{@file_name}" if qiter.null?
+
+      query_reuse_yield_vcf(qiter, &block)
+      self
+    end
+
+    def query_reuse_yield_vcf(qiter)
+      line = LibHTS::KString.new
+      bcf1 = LibHTS.bcf_init
+      record = Record.new(header, bcf1)
+      begin
+        while (slen = LibHTS.tbx_itr_next(@hts_file, @idx, qiter, line)) >= 0
+          raise QueryError, "Failed to parse VCF record in #{@file_name}" if LibHTS.vcf_parse(line, read_header,
+                                                                                              bcf1) < 0
+
+          apply_subset!(record)
+          yield record
+        end
+        raise if slen < -1
+      ensure
+        line.free_buffer
+        LibHTS.hts_itr_destroy(qiter)
+      end
+    end
+
+    def queryi_copy_vcf(tid, beg, end_, &block)
+      qiter = LibHTS.tbx_itr_queryi(@idx, tid, beg, end_)
+      raise QueryError, "Failed to query region #{tid}:#{beg}-#{end_} in #{@file_name}" if qiter.null?
+
+      query_copy_yield_vcf(qiter, &block)
+      self
+    end
+
+    def querys_copy_vcf(region, &block)
+      qiter = LibHTS.tbx_itr_querys(@idx, region)
+      raise QueryError, "Failed to query region #{region.inspect} in #{@file_name}" if qiter.null?
+
+      query_copy_yield_vcf(qiter, &block)
+      self
+    end
+
+    def query_copy_yield_vcf(qiter)
+      line = LibHTS::KString.new
+      begin
+        while (slen = LibHTS.tbx_itr_next(@hts_file, @idx, qiter, line)) >= 0
+          bcf1 = LibHTS.bcf_init
+          raise QueryError, "Failed to parse VCF record in #{@file_name}" if LibHTS.vcf_parse(line, read_header,
+                                                                                              bcf1) < 0
+
+          record = Record.new(header, bcf1)
+          apply_subset!(record)
+          yield record
+        end
+        raise if slen < -1
+      ensure
+        line.free_buffer
+        LibHTS.hts_itr_destroy(qiter)
+      end
+    end
+
     def each_record_reuse
       check_closed
 
@@ -352,7 +463,10 @@ module HTS
 
       bcf1 = LibHTS.bcf_init
       record = Record.new(header, bcf1)
-      yield record while LibHTS.bcf_read(@hts_file, header, bcf1) != -1
+      while LibHTS.bcf_read(@hts_file, read_header, bcf1) != -1
+        apply_subset!(record)
+        yield record
+      end
       self
     end
 
@@ -361,11 +475,26 @@ module HTS
 
       return to_enum(__method__) unless block_given?
 
-      while LibHTS.bcf_read(@hts_file, header, bcf1 = LibHTS.bcf_init) != -1
-        record = Record.new(header, bcf1)
-        yield record
+      bcf1 = LibHTS.bcf_init
+      record = Record.new(header, bcf1)
+      while LibHTS.bcf_read(@hts_file, read_header, bcf1) != -1
+        apply_subset!(record)
+        yield record.dup
       end
       self
     end
+
+    def read_header
+      @read_header || header
+    end
+
+    def apply_subset!(record)
+      return unless header.subset?
+
+      rc = LibHTS.bcf_subset(header.struct, record.struct, header.subset_sample_count, header.subset_imap_pointer || ::FFI::Pointer::NULL)
+      return if rc >= 0
+
+      raise SubsetError, "Failed to subset samples #{header.subset_samples.inspect} while reading #{@file_name}"
+    end
   end
 end

data/lib/hts/faidx.rb

@@ -1,14 +1,21 @@
 # frozen_string_literal: true
 
 require_relative "../htslib"
-require_relative "faidx/sequence"
+
+module HTS
+  module LibC
+    extend FFI::Library
+    ffi_lib FFI::Library::LIBC
+    attach_function :free, [:pointer], :void
+  end
+end
 
 module HTS
   class Faidx
-    attr_reader :file_name
+    attr_reader :file_name, :format
 
-    def self.open(*args, **kw)
-      file = new(*args, **kw) # do not yield
+    def self.open(file_name, format: :auto, auto_build: true)
+      file = new(file_name, format:, auto_build:) # do not yield
       return file unless block_given?
 
       begin
@@ -19,19 +26,19 @@ module HTS
       file
     end
 
-    def initialize(file_name)
-      if block_given?
-        message = "HTS::Faidx.new() does not take block; Please use HTS::Faidx.open() instead"
-        raise message
+    def self.build_index(file_name, fai_path = nil, gzi_path = nil)
+      case LibHTS.fai_build3(file_name, fai_path, gzi_path)
+      when 0
+      else raise HTS::Error, "Failed to build faidx index for #{file_name}"
       end
+    end
+
+    def initialize(file_name, format: :auto, auto_build: true)
+      raise ArgumentError, "HTS::Faidx.new() does not take block; Please use HTS::Faidx.open() instead" if block_given?
 
-      @file_name = file_name
-      @fai = case File.extname(@file_name)
-             when ".fq", ".fastq"
-               LibHTS.fai_load_format(@file_name, 2)
-             else
-               LibHTS.fai_load(@file_name)
-             end
+      @file_name = file_name.freeze
+      @format = resolve_format(@file_name, format)
+      @fai = load_handle(@file_name, @format, auto_build)
 
       raise Errno::ENOENT, "Failed to open #{@file_name}" if @fai.null?
     end
@@ -51,106 +58,136 @@ module HTS
       @fai.nil? || @fai.null?
     end
 
-    def file_format
-      @fai[:format]
-    end
-
-    # the number of sequences in the index.
-    def length
+    def size
+      check_closed
       LibHTS.faidx_nseq(@fai)
     end
-    alias size length
 
-    # return the length of the requested chromosome.
+    alias length size
+
     def names
+      check_closed
       Array.new(length) { |i| LibHTS.faidx_iseq(@fai, i) }
     end
 
-    alias keys names
-
-    def has_key?(key)
+    def has_seq?(key)
+      check_closed
       raise ArgumentError, "Expect chrom to be String or Symbol" unless key.is_a?(String) || key.is_a?(Symbol)
 
       key = key.to_s
       case LibHTS.faidx_has_seq(@fai, key)
       when 1 then true
       when 0 then false
-      else raise
+      else raise HTS::Error, "Unexpected return value from faidx_has_seq"
       end
     end
 
-    def [](name)
-      name = LibHTS.faidx_iseq(@fai, name) if name.is_a?(Integer)
-      Sequence.new(self, name)
-    end
-
-    # return the length of the requested chromosome.
     def seq_len(chrom)
+      check_closed
       raise ArgumentError, "Expect chrom to be String or Symbol" unless chrom.is_a?(String) || chrom.is_a?(Symbol)
 
       chrom = chrom.to_s
-      result = LibHTS.faidx_seq_len(@fai, chrom)
-      result == -1 ? nil : result
-    end
+      result = LibHTS.faidx_seq_len64(@fai, chrom)
+      raise ArgumentError, "Sequence not found: #{chrom}" if result == -1
 
-    # @overload seq(name)
-    #   Fetch the sequence as a String.
-    #   @param name [String] chr1:0-10
-    # @overload seq(name, start, stop)
-    #   Fetch the sequence as a String.
-    #   @param name [String] the name of the chromosome
-    #   @param start [Integer] the start position of the sequence (0-based)
-    #   @param stop [Integer] the end position of the sequence (0-based)
-    #   @return [String] the sequence
+      result
+    end
 
     def fetch_seq(name, start = nil, stop = nil)
+      check_closed
       name = name.to_s
-      rlen = FFI::MemoryPointer.new(:int)
 
       if start.nil? && stop.nil?
-        result = LibHTS.fai_fetch64(@fai, name, rlen)
-      else
-        start < 0    && raise(ArgumentError, "Expect start to be >= 0")
-        stop  < 0    && raise(ArgumentError, "Expect stop to be >= 0")
-        start > stop && raise(ArgumentError, "Expect start to be <= stop")
-        stop >= seq_len(name) && raise(ArgumentError, "Expect stop to be < seq_len")
+        len = seq_len(name)
+        return "" if len.zero?
 
+        fetch_seq(name, 0, len - 1)
+      else
+        validate_range!(name, start, stop)
+        rlen = FFI::MemoryPointer.new(:int64)
         result = LibHTS.faidx_fetch_seq64(@fai, name, start, stop, rlen)
+        fetch_result(result, rlen.read_int64, "sequence", name, start, stop)
       end
-
-      case rlen.read_int
-      when -2 then raise "Invalid chromosome name: #{name}"
-      when -1 then raise "Error fetching sequence: #{name}:#{start}-#{stop}"
-      end
-
-      result
     end
 
-    alias seq fetch_seq
-
     def fetch_qual(name, start = nil, stop = nil)
+      check_closed
+      raise HTS::Error, "Quality is only available for FASTQ indexes" unless format == :fastq
+
       name = name.to_s
-      rlen = FFI::MemoryPointer.new(:int)
 
       if start.nil? && stop.nil?
-        result = LibHTS.fai_fetchqual64(@fai, name, rlen)
-      else
-        start < 0    && raise(ArgumentError, "Expect start to be >= 0")
-        stop  < 0    && raise(ArgumentError, "Expect stop to be >= 0")
-        start > stop && raise(ArgumentError, "Expect start to be <= stop")
-        stop >= seq_len(name) && raise(ArgumentError, "Expect stop to be < seq_len")
+        len = seq_len(name)
+        return "" if len.zero?
 
+        fetch_qual(name, 0, len - 1)
+      else
+        validate_range!(name, start, stop)
+        rlen = FFI::MemoryPointer.new(:int64)
         result = LibHTS.faidx_fetch_qual64(@fai, name, start, stop, rlen)
+        fetch_result(result, rlen.read_int64, "quality", name, start, stop)
       end
+    end
+
+    def build_index(fai_path = nil, gzi_path = nil)
+      self.class.build_index(@file_name, fai_path, gzi_path)
+      self
+    end
+
+    private
 
-      case rlen.read_int
-      when -2 then raise "Invalid chromosome name: #{name}"
-      when -1 then raise "Error fetching sequence: #{name}:#{start}-#{stop}"
+    def check_closed
+      raise IOError, "closed Faidx" if closed?
+    end
+
+    def validate_range!(name, start, stop)
+      raise ArgumentError, "Expect start to be >= 0" if start < 0
+      raise ArgumentError, "Expect stop to be >= 0" if stop < 0
+      raise ArgumentError, "Expect start to be <= stop" if start > stop
+
+      len = seq_len(name)
+      raise ArgumentError, "Expect stop to be < seq_len (#{len})" if stop >= len
+    end
+
+    def fetch_result(ptr, len, kind, name, start, stop)
+      case len
+      when -2 then raise ArgumentError, "Sequence not found: #{name}"
+      when -1 then raise HTS::Error, "Error fetching #{kind}: #{name}:#{start}-#{stop}"
       end
 
-      result
+      raise HTS::Error, "Error fetching #{kind}: #{name}:#{start}-#{stop}" if ptr.null?
+
+      begin
+        ptr.read_string_length(len)
+      ensure
+        HTS::LibC.free(ptr)
+      end
     end
 
-    alias qual fetch_qual
+    def load_handle(file_name, format, auto_build)
+      case [format, auto_build]
+      when [:fasta, true]
+        LibHTS.fai_load_format(file_name, :FAI_FASTA)
+      when [:fastq, true]
+        LibHTS.fai_load_format(file_name, :FAI_FASTQ)
+      when [:fasta, false]
+        LibHTS.fai_load3_format(file_name, nil, nil, 0, :FAI_FASTA)
+      when [:fastq, false]
+        LibHTS.fai_load3_format(file_name, nil, nil, 0, :FAI_FASTQ)
+      else
+        raise ArgumentError, "Unsupported format: #{format}"
+      end
+    end
+
+    def resolve_format(file_name, format)
+      case format
+      when :auto
+        file_name.match?(/\.(fastq|fq)(\.gz|\.bgz)?\z/i) ? :fastq : :fasta
+      when :fasta, :fastq
+        format
+      else
+        raise ArgumentError, "Unsupported format: #{format}"
+      end
+    end
   end
 end

data/lib/hts/hts.rb

@@ -71,7 +71,10 @@ module HTS
 
     def fai=(fai)
       check_closed
-      LibHTS.hts_set_fai_filename(@hts_file, fai) > 0 || raise
+      r = LibHTS.hts_set_fai_filename(@hts_file, fai)
+      raise "Failed to load fasta index: #{fai}" if r.negative?
+
+      self
     end
 
     def set_threads(n = nil)

data/lib/hts/libhts/constants.rb

@@ -12,6 +12,27 @@ module HTS
         :l,              :size_t,
         :m,              :size_t,
         :s,              :string
+
+      def buffer_ptr
+        to_ptr.get_pointer(self.class.offset_of(:s))
+      end
+
+      def read_string_copy
+        ptr = buffer_ptr
+        return "" if ptr.null?
+
+        ptr.read_string(self[:l])
+      end
+
+      def free_buffer
+        ptr = buffer_ptr
+        return if ptr.null?
+
+        LibHTS.hts_free(ptr)
+        to_ptr.put_pointer(self.class.offset_of(:s), FFI::Pointer::NULL)
+        self[:l] = 0
+        self[:m] = 0
+      end
     end
 
     class KSeq < FFI::Struct
@@ -430,6 +451,17 @@ module HTS
         :idx,            HtsIdx.ptr,
         :dict,           :pointer
 
+      def close
+        return if @closed
+
+        ptr = to_ptr
+        unless ptr.null?
+          ptr.autorelease = false if ptr.respond_to?(:autorelease=)
+          self.class.release(ptr)
+        end
+        @closed = true
+      end
+
       def self.release(ptr)
         LibHTS.tbx_destroy(ptr) unless ptr.null?
       end
@@ -439,7 +471,13 @@ module HTS
 
     FaiFormatOptions = enum(:FAI_NONE, :FAI_FASTA, :FAI_FASTQ)
 
-    class Faidx < FFI::Struct # FIXME: ManagedStruct
+    # Faidx represents a faidx_t handle which is treated as a
+    # file-level RAII object in HTS::Faidx. It is intentionally
+    # kept as a plain Struct and is destroyed explicitly via
+    # LibHTS.fai_destroy in HTS::Faidx#close. Do not convert this
+    # to ManagedStruct; that would interfere with the explicit
+    # lifetime managed by the Ruby wrapper.
+    class Faidx < FFI::Struct
       layout :bgzf,      BGZF.ptr,
              :n,         :int,
              :m,         :int,
@@ -460,7 +498,7 @@ module HTS
         :n,              :int
     end
 
-    # Complete textual representation of a header line
+    # Complete textual representation of a header line owned by Ruby.
     class BcfHrec < FFI::ManagedStruct
       layout \
         :type,           :int,
@@ -497,7 +535,7 @@ module HTS
     class BcfIdinfo < FFI::Struct
       layout \
         :info,           [:uint64, 3],
-        :hrec,           [BcfHrec.ptr, 3],
+        :hrec,           [:pointer, 3],
         :id,             :int
     end
 

data/lib/hts/libhts/cram.rb

@@ -330,11 +330,6 @@ module HTS
       %i[cram_fd cram_block cram_metrics int int],
       :int
 
-    # attach_function \
-    #   :cram_compress_block2,
-    #   %i[cram_fd cram_slice cram_block cram_metrics int int],
-    #   :int
-
     # Creates a new container, specifying the maximum number of slices
     # and records permitted.
     attach_function \