htslib 0.4.2 → 0.5.0

data/lib/hts/bam/mpileup.rb

@@ -2,173 +2,199 @@
 
 module HTS
   class Bam < Hts
-    # High-level mpileup iterator over multiple BAM/CRAM inputs
     class Mpileup
       include Enumerable
 
-      # Usage:
-      #   HTS::Bam::Mpileup.open([bam1, bam2], region: "chr1:1-100") do |mpl|
-      #     mpl.each { |cols| ... }
-      #   end
-      def self.open(*args, **kw)
-        m = new(*args, **kw)
-        return m unless block_given?
+      class DepthView
+        include Enumerable
+        attr_reader :length
+        alias size length
+        def initialize = reset([])
+
+        def reset(values)
+          @values = values
+          @length = values.length
+          self
+        end
+
+        def [](index)
+          index = Integer(index)
+          index += @length if index.negative?
+          raise IndexError, "depth index #{index} outside of view" unless index.between?(0, @length - 1)
+
+          @values[index]
+        end
+
+        def each(&block)
+          return to_enum(__method__) unless block_given?
+
+          @values.each(&block)
+          self
+        end
+      end
+
+      class ColumnsView
+        include Enumerable
+        attr_reader :tid, :pos, :length
+        alias size length
+        def initialize = @column_view = Pileup::BorrowedColumnView.new
+
+        def reset(tid, pos, rows)
+          @tid = tid
+          @pos = pos
+          @rows = rows
+          @length = rows.length
+          self
+        end
+
+        def each
+          return to_enum(__method__) unless block_given?
+
+          @rows.each_with_index { |entries, index| yield index, @column_view.reset(entries, @tid, @pos) }
+          self
+        end
+      end
+
+      def self.open(*args, **keywords)
+        mpileup = new(*args, **keywords)
+        return mpileup unless block_given?
 
         begin
-          yield m
+          yield mpileup
         ensure
-          m.close
+          mpileup.close
         end
-        m
+        mpileup
       end
 
-      # Normalize inputs to HTS::Bam instances
-      # Accepts array of HTS::Bam or filenames (String)
       def initialize(inputs, region: nil, beg: nil, end_: nil, maxcnt: nil, overlaps: false)
         raise ArgumentError, "inputs must be non-empty" if inputs.nil? || inputs.empty?
 
-        @owned_bams = [] # Bams we opened here; will be closed on close
-        @bams = inputs.map do |x|
-          case x
-          when HTS::Bam
-            x
+        @owned_bams = []
+        @bams = inputs.map do |input|
+          case input
+          when HTS::Bam then input
           when String
-            b = HTS::Bam.open(x)
-            @owned_bams << b
-            b
-          else
-            raise ArgumentError, "Unsupported input type: #{x.class}"
+            HTS::Bam.open(input).tap { |bam| @owned_bams << bam }
+          else raise ArgumentError, "Unsupported input type: #{input.class}"
           end
         end
-
-        n = @bams.length
-        @iters       = []
-        @data_blocks = [] # per-input packed pointers kept alive
-        @data_entries = {}
-
-        # Prepare optional region iterators for each input
-        @bams.each_with_index do |bam, i|
-          itr = nil
-          if region && beg.nil? && end_.nil?
-            raise "Index required for region mpileup" unless bam.index_loaded?
-
-            itr = HTS::LibHTS.sam_itr_querys(bam.instance_variable_get(:@idx), bam.header.struct, region)
-            raise "Failed to query region on input ##{i}: #{region}" if itr.null?
-          elsif region && beg && end_
-            raise "Index required for region mpileup" unless bam.index_loaded?
-
-            tid = bam.header.get_tid(region)
-            itr = HTS::LibHTS.sam_itr_queryi(bam.instance_variable_get(:@idx), tid, beg, end_)
-            raise "Failed to query region on input ##{i}: #{region} #{beg} #{end_}" if itr.null?
-          elsif beg || end_
-            raise ArgumentError, "beg and end_ must be specified together"
-          end
-          @iters << itr
+        if region
+          missing = @bams.find { |bam| !bam.__send__(:ensure_index_loaded) }
+          raise "Index file is required to use region mpileup: #{missing.file_name}" if missing
         end
+        files = @bams.map { |bam| bam.__send__(:native_handle) }
+        headers = @bams.map { |bam| bam.header.__send__(:native_handle) }
+        @native = Native::MpileupHandle.open(files, headers, region, beg, end_, maxcnt, overlaps)
+        @closed = false
+      end
 
-        # Build per-input packed pointer blocks so C passes them back to the callback.
-        # Keep the Ruby FFI structs in @data_entries to avoid rebuilding wrappers
-        # in the per-record callback.
-        ptr_size = FFI.type_size(:pointer)
-        data_array = FFI::MemoryPointer.new(:pointer, n)
-        @bams.each_with_index do |bam, i|
-          hts_fp     = bam.instance_variable_get(:@hts_file)
-          hdr_struct = bam.header.struct
-          itr        = @iters[i]
-          block = FFI::MemoryPointer.new(:pointer, 3)
-          block.put_pointer(0 * ptr_size, hts_fp)
-          block.put_pointer(1 * ptr_size, hdr_struct)
-          block.put_pointer(2 * ptr_size, itr && !itr.null? ? itr : FFI::Pointer::NULL)
-          @data_blocks << block
-          @data_entries[block.address] = [hts_fp, hdr_struct, itr && !itr.null? ? itr : nil]
-          data_array.put_pointer(i * ptr_size, block)
-        end
-        # Keep the array of per-input blocks alive while the C side holds on to them
-        @data_array = data_array
-
-        @cb = FFI::Function.new(:int, %i[pointer pointer]) do |data, b|
-          hts_fp, hdr_struct, itr = @data_entries.fetch(data.address)
-          # HTSlib contract: return same as sam_itr_next/sam_read1 (>= 0 on success, -1 on EOF, < -1 on error)
-          if itr
-            HTS::LibHTS.sam_itr_next(hts_fp, itr, b)
-          else
-            HTS::LibHTS.sam_read1(hts_fp, hdr_struct, b)
+      def each
+        return to_enum(__method__) unless block_given?
+
+        headers = @bams.map(&:header)
+        each_column_raw do |tid, pos, _depths, rows_by_input, _|
+          columns = rows_by_input.each_with_index.map do |rows, index|
+            alignments = rows.map { |row| Pileup::PileupRecord.new(row, headers[index]) }
+            Pileup::PileupColumn.new(tid:, pos:, alignments:)
           end
+          yield columns
         end
+        self
+      end
 
-        @iter = HTS::LibHTS.bam_mplp_init(n, @cb, @data_array)
-        raise "bam_mplp_init failed" if @iter.null?
+      def each_depth
+        return to_enum(__method__) unless block_given?
 
-        HTS::LibHTS.bam_mplp_set_maxcnt(@iter, maxcnt) if maxcnt
-        return unless overlaps
+        view = DepthView.new
+        each_column_raw { |tid, pos, depths, _, _| yield tid, pos, view.reset(depths) }
+        self
+      end
 
-        rc = HTS::LibHTS.bam_mplp_init_overlaps(@iter)
-        raise "bam_mplp_init_overlaps failed" if rc < 0
+      def each_view
+        return to_enum(__method__) unless block_given?
+
+        view = ColumnsView.new
+        each_column_raw { |tid, pos, _, rows, _| yield view.reset(tid, pos, rows) }
+        self
       end
 
-      # Yields an array of Pileup::PileupColumn (one per input) for each position
-      def each
+      def each_entry_raw
         return to_enum(__method__) unless block_given?
 
-        n = @bams.length
-        tid_ptr = FFI::MemoryPointer.new(:int)
-        pos_ptr = FFI::MemoryPointer.new(:long_long)
-        n_ptr   = FFI::MemoryPointer.new(:int, n)
-        plp_ptr = FFI::MemoryPointer.new(:pointer, n)
-        plp1_size = HTS::LibHTS::BamPileup1.size
-        headers   = @bams.map(&:header)
-
-        while HTS::LibHTS.bam_mplp64_auto(@iter, tid_ptr, pos_ptr, n_ptr, plp_ptr) > 0
-          tid = tid_ptr.read_int
-          pos = pos_ptr.read_long_long
-
-          counts = n_ptr.read_array_of_int(n)
-          plp_arr = plp_ptr.read_array_of_pointer(n)
-
-          cols = Array.new(n)
-          i = 0
-          while i < n
-            c = counts[i]
-            if c <= 0 || plp_arr[i].null?
-              cols[i] = HTS::Bam::Pileup::PileupColumn.new(tid: tid, pos: pos, alignments: [])
-            else
-              base_ptr = plp_arr[i]
-              aligns = Array.new(c)
-              j = 0
-              while j < c
-                e_ptr = base_ptr + (j * plp1_size)
-                entry = HTS::LibHTS::BamPileup1.new(e_ptr)
-                aligns[j] = HTS::Bam::Pileup::PileupRecord.new(entry, headers[i])
-                j += 1
-              end
-              cols[i] = HTS::Bam::Pileup::PileupColumn.new(tid: tid, pos: pos, alignments: aligns)
-            end
-            i += 1
+        each_column_raw do |tid, pos, _, rows_by_input, _|
+          rows_by_input.each_with_index do |rows, input_index|
+            rows.each { |row| yield input_index, tid, pos, row[1], row[7], row[8], row[9] }
           end
-
-          yield cols
         end
-
         self
       end
 
-      def close
-        if @iter && !@iter.null?
-          HTS::LibHTS.bam_mplp_destroy(@iter)
-          @iter = FFI::Pointer::NULL
+      def each_base_counts(min_base_quality: 0, min_mapping_quality: 0)
+        return to_enum(__method__, min_base_quality:, min_mapping_quality:) unless block_given?
+
+        min_base_quality = Integer(min_base_quality)
+        min_mapping_quality = Integer(min_mapping_quality)
+        if min_base_quality.negative? || min_mapping_quality.negative?
+          raise ArgumentError,
+                "quality thresholds must be non-negative"
         end
-        @iters.each do |itr|
-          HTS::LibHTS.hts_itr_destroy(itr) if itr && !itr.null?
+
+        counts = Array.new(@bams.length) { Array.new(Pileup::BASE_COUNT_FIELDS.length, 0) }
+        each_column_raw do |tid, pos, _, rows_by_input, _|
+          rows_by_input.each_with_index do |rows, input_index|
+            aggregate_counts(rows, counts[input_index], min_base_quality, min_mapping_quality)
+          end
+          yield tid, pos, counts
         end
-        @iters.clear
-        # Keep references to callback and data blocks to prevent GC
-        @_keepalive = [@cb, @data_array, @data_entries, *@data_blocks]
-        # Close owned bams opened by this object
-        @owned_bams.each do |b|
-          b.close
-        rescue StandardError
+        self
+      end
+
+      # The fourth value is an Array of opaque native-entry rows. It remains
+      # borrowed until the next iteration.
+      def each_column_raw
+        return to_enum(__method__) unless block_given?
+
+        while (column = @native.next)
+          tid, pos, depths, rows = column
+          yield tid, pos, depths, rows, @bams.length
         end
+        self
+      end
+
+      def reset
+        @native.reset
+        self
+      end
+
+      def close
+        return if @closed
+
+        @native.close
+        @owned_bams.each(&:close)
         @owned_bams.clear
+        @closed = true
+        nil
+      end
+
+      private
+
+      def aggregate_counts(rows, counts, min_base_quality, min_mapping_quality)
+        counts.fill(0)
+        rows.each do |row|
+          next if row[6] || row[10] < min_mapping_quality
+
+          if row[3] || row[1].negative?
+            counts[8] += 1
+          else
+            next if row[9] < min_base_quality
+
+            counts[{ 1 => 1, 2 => 2, 4 => 3, 8 => 4 }.fetch(row[8], 5)] += 1
+          end
+          counts[0] += 1
+          counts[(row[7] & 16).zero? ? 6 : 7] += 1
+          counts[9] += 1 if row[2].positive?
+        end
       end
     end
   end

data/lib/hts/bam/pileup.rb

@@ -1,200 +1,175 @@
 # frozen_string_literal: true
 
+require_relative "../native"
+
 module HTS
   class Bam < Hts
-    # High-level pileup iterator for a single SAM/BAM/CRAM
     class Pileup
       include Enumerable
+      BASE_COUNT_FIELDS = %i[depth a c g t n forward reverse deletion insertion].freeze
 
-      # Usage:
-      #   HTS::Bam::Pileup.open(bam, region: "chr1:1-100") do |pl|
-      #     pl.each { |col| ... }
-      #   end
-      def self.open(*args, **kw)
-        pu = new(*args, **kw)
-        return pu unless block_given?
+      def self.open(*args, **keywords)
+        pileup = new(*args, **keywords)
+        return pileup unless block_given?
 
         begin
-          yield pu
+          yield pileup
         ensure
-          pu.close
+          pileup.close
         end
-        pu
+        pileup
       end
 
-      # A column at a reference position with pileup alignments
       PileupColumn = Struct.new(:tid, :pos, :alignments, keyword_init: true) do
-        def depth
-          alignments.length
-        end
+        def depth = alignments.length
       end
 
-      # A wrapper of one bam_pileup1_t entry
       class PileupRecord
-        def initialize(entry, header)
-          @entry  = entry
+        def initialize(values, header)
+          @values = values
           @header = header
-          @record = nil
         end
 
-        # Return Bam::Record. On the first call, duplicate the underlying bam1_t (bam_dup1)
-        # so the record becomes safe to keep beyond the current pileup step. Subsequent calls
-        # return the cached Bam::Record instance.
-        # NOTE: Without duplication, bam1_t memory may be reused by HTSlib on the next step.
-        def record
-          return @record if @record
-
-          # Normalize to a raw pointer and duplicate to obtain owned memory.
-          b_ptr = @entry[:b].is_a?(FFI::Pointer) ? @entry[:b] : @entry[:b].to_ptr
-          dup_ptr = HTS::LibHTS.bam_dup1(b_ptr)
-          raise "bam_dup1 failed" if dup_ptr.null?
-
-          # Build a Bam::Record backed by the duplicated bam1_t.
-          @record = HTS::Bam::Record.new(@header, dup_ptr)
-        end
+        def record = (@record ||= HTS::Bam::Record.new(@header, @values[0]))
+        def query_position = @values[1]
+        def indel = @values[2]
+        def del? = @values[3]
+        def head? = @values[4]
+        def tail? = @values[5]
+        def refskip? = @values[6]
+      end
 
-        def query_position
-          @entry[:qpos]
+      class BorrowedEntryView
+        def reset(values, tid, pos)
+          @values = values
+          @tid = tid
+          @pos = pos
+          self
         end
+        attr_reader :tid, :pos
+
+        def query_position = @values[1]
+        def indel = @values[2]
+        def del? = @values[3]
+        def refskip? = @values[6]
+        def flag = @values[7]
+        def base_code = @values[8]
+        def quality = @values[9]
+      end
 
-        def indel
-          @entry[:indel]
-        end
+      class BorrowedColumnView
+        include Enumerable
+        attr_reader :tid, :pos, :depth
 
-        def del?
-          @entry[:is_del] == 1
-        end
+        def initialize = @entry_view = BorrowedEntryView.new
 
-        def head?
-          @entry[:is_head] == 1
+        def reset(rows, tid, pos)
+          @rows = rows
+          @tid = tid
+          @pos = pos
+          @depth = rows.length
+          self
         end
 
-        def tail?
-          @entry[:is_tail] == 1
-        end
+        def each
+          return to_enum(__method__) unless block_given?
 
-        def refskip?
-          @entry[:is_refskip] == 1
+          @rows.each { |row| yield @entry_view.reset(row, @tid, @pos) }
+          self
         end
       end
 
-      # Create a Pileup iterator
-      # @param bam [HTS::Bam]
-      # @param region [String, nil] Optional region string (requires index)
-      # @param beg [Integer, nil] Optional begin when using tid/beg/end form
-      # @param end_ [Integer, nil] Optional end when using tid/beg/end form
-      # @param maxcnt [Integer, nil] Max per-position depth (capped)
       def initialize(bam, region: nil, beg: nil, end_: nil, maxcnt: nil)
-        @bam    = bam
+        raise ArgumentError, "beg and end_ must be specified together" if beg.nil? != end_.nil?
+        raise ArgumentError, "region is required when beg/end_ are specified" if !beg.nil? && region.nil?
+
+        @bam = bam
         @header = bam.header
-        @itr    = nil
-        @cb     = nil
-        @plp    = nil
-
-        # Optional region iterator
-        if region && beg.nil? && end_.nil?
-          raise "Index file is required to use region pileup." unless bam.index_loaded?
-
-          @itr = HTS::LibHTS.sam_itr_querys(bam.instance_variable_get(:@idx), @header.struct, region)
-          raise "Failed to query region: #{region}" if @itr.null?
-        elsif region && beg && end_
-          raise "Index file is required to use region pileup." unless bam.index_loaded?
-
-          tid = @header.get_tid(region)
-          @itr = HTS::LibHTS.sam_itr_queryi(bam.instance_variable_get(:@idx), tid, beg, end_)
-          raise "Failed to query region: #{region} #{beg} #{end_}" if @itr.null?
-        elsif beg || end_
-          raise ArgumentError, "beg and end_ must be specified together"
+        if region && !bam.__send__(:ensure_index_loaded)
+          raise "Index file is required to use region pileup."
         end
-
-        # Build the auto callback for bam_plp_init (micro-optimized)
-        # - Hoist ivar/constant lookups out of the callback to reduce per-call overhead.
-        # - Specialize callbacks to avoid branching in the hot path.
-        hts_fp     = @bam.instance_variable_get(:@hts_file)
-        hdr_struct = @header.struct
-        itr_local  = @itr
-
-        @cb = if itr_local && !itr_local.null?
-                FFI::Function.new(:int, %i[pointer pointer]) do |_data, b|
-                  # HTSlib contract: return same as sam_itr_next (>= 0 on success, -1 on EOF, < -1 on error)
-                  HTS::LibHTS.sam_itr_next(hts_fp, itr_local, b)
-                end
-              else
-                FFI::Function.new(:int, %i[pointer pointer]) do |_data, b|
-                  # HTSlib contract: return same as sam_read1 (>= 0 on success, -1 on EOF, < -1 on error)
-                  HTS::LibHTS.sam_read1(hts_fp, hdr_struct, b)
-                end
-              end
-
-        @plp = HTS::LibHTS.bam_plp_init(@cb, nil)
-        raise "bam_plp_init failed" if @plp.null?
-
-        HTS::LibHTS.bam_plp_set_maxcnt(@plp, maxcnt) if maxcnt
+        @native = Native::PileupHandle.open(
+          bam.__send__(:native_handle), @header.__send__(:native_handle), region, beg, end_, maxcnt
+        )
       end
 
       def each
         return to_enum(__method__) unless block_given?
 
-        tid_ptr = FFI::MemoryPointer.new(:int)
-        pos_ptr = FFI::MemoryPointer.new(:long_long) # hts_pos_t
-        n_ptr   = FFI::MemoryPointer.new(:int)
+        each_raw_column do |tid, pos, rows|
+          alignments = rows.map { |row| PileupRecord.new(row, @header) }
+          yield PileupColumn.new(tid:, pos:, alignments:)
+        end
+        self
+      end
 
-        # Micro-optimizations:
-        # - Compute constant struct size once
-        # - Hoist header reference outside the loop
-        plp1_size    = HTS::LibHTS::BamPileup1.size
-        header_local = @header
+      def each_depth
+        return to_enum(__method__) unless block_given?
 
-        loop do
-          base_ptr = HTS::LibHTS.bam_plp64_auto(@plp, tid_ptr, pos_ptr, n_ptr)
+        each_raw_column { |tid, pos, rows| yield tid, pos, rows.length }
+        self
+      end
 
-          # When base_ptr is NULL, check n to distinguish EOF (n == 0) from error (n < 0)
-          if base_ptr.null?
-            n = n_ptr.read_int
-            raise "HTSlib pileup error (bam_plp64_auto)" if n < 0
+      def each_view
+        return to_enum(__method__) unless block_given?
 
-            break
-          end
+        view = BorrowedColumnView.new
+        each_raw_column { |tid, pos, rows| yield view.reset(rows, tid, pos) }
+        self
+      end
 
-          tid = tid_ptr.read_int
-          pos = pos_ptr.read_long_long
-          n   = n_ptr.read_int
-
-          # Construct alignment entries with minimal allocations
-          if n.zero?
-            alignments = []
-          else
-            alignments = Array.new(n)
-            i = 0
-            while i < n
-              e_ptr = base_ptr + (i * plp1_size)
-              entry = HTS::LibHTS::BamPileup1.new(e_ptr)
-              alignments[i] = PileupRecord.new(entry, header_local)
-              i += 1
-            end
-          end
+      def each_entry_raw
+        return to_enum(__method__) unless block_given?
 
-          yield PileupColumn.new(tid: tid, pos: pos, alignments: alignments)
+        each_raw_column do |tid, pos, rows|
+          rows.each { |row| yield tid, pos, row[1], row[7], row[8], row[9] }
         end
-
         self
       end
 
-      def reset
-        HTS::LibHTS.bam_plp_reset(@plp) if @plp && !@plp.null?
-      end
+      def each_base_counts(min_base_quality: 0, min_mapping_quality: 0)
+        return to_enum(__method__, min_base_quality:, min_mapping_quality:) unless block_given?
 
-      def close
-        if @plp && !@plp.null?
-          HTS::LibHTS.bam_plp_destroy(@plp)
-          @plp = FFI::Pointer::NULL
+        min_base_quality = Integer(min_base_quality)
+        min_mapping_quality = Integer(min_mapping_quality)
+        if min_base_quality.negative? || min_mapping_quality.negative?
+          raise ArgumentError,
+                "quality thresholds must be non-negative"
         end
-        if @itr && !@itr.null?
-          HTS::LibHTS.hts_itr_destroy(@itr)
-          @itr = FFI::Pointer::NULL
+
+        counts = Array.new(BASE_COUNT_FIELDS.length, 0)
+        each_raw_column do |tid, pos, rows|
+          counts.fill(0)
+          rows.each do |row|
+            next if row[6] || row[10] < min_mapping_quality
+
+            if row[3] || row[1].negative?
+              counts[8] += 1
+            else
+              next if row[9] < min_base_quality
+
+              counts[{ 1 => 1, 2 => 2, 4 => 3, 8 => 4 }.fetch(row[8], 5)] += 1
+            end
+            counts[0] += 1
+            counts[(row[7] & 16).zero? ? 6 : 7] += 1
+            counts[9] += 1 if row[2].positive?
+          end
+          yield tid, pos, counts
+        end
+        self
+      end
+
+      def reset = @native.reset
+      def close = @native&.close
+
+      private
+
+      def next_raw = @native.next
+
+      def each_raw_column
+        while (column = next_raw)
+          yield(*column)
         end
-        # Keep @cb referenced by instance to avoid GC during iteration.
-        @cb
       end
     end
   end