htslib 0.4.2 → 0.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 69e910e95cfe1bef250416f8bbe480a39a1bc54b53103435cf15811860f3b92d
-  data.tar.gz: 060c7e5af8127c496af3b5d5bec5eca5e7476cb34ccc86a75546fd1a80d6dd8a
+  metadata.gz: eb6716391f6e8aa27cb4859140deb80073a7b02e149f17704a4b08e125567cb9
+  data.tar.gz: 20ec9fc316dd378bd869d590360fc8d50a9b2be075871e76a1b4148292614cf6
 SHA512:
-  metadata.gz: 62adc672a3606265d231efa7f568232d55279634c8fb650190aee01c750b0dbac2421a1cf8fb2507bf5b21e16afbe4e4425354cd3dd2b1398d4df7b3f24e6655
-  data.tar.gz: 2a6bd47a73ff2695a515e934fa1919d8fcdf1108e10178404af31da769b4e7c40a8580f2382ab8f7e6ac8697261344fb32ecebc138807d5cd03724745457f4d4
+  metadata.gz: ed55104f781970f32e5918d15fbe2d0cf5c08e01524c29cf95656b6da902a1528f7630152ae8eb9adee815442230e74868707d348bc7947a871b9576d44bcf0d
+  data.tar.gz: c21d82a89b123e25e135486844d09ba83734fb1149269a158b1f2b81be94708ba3e55f95ee94d3a54e78b40a40db7ebcc71fdce969ecc16950802f84bee79461

data/README.md

@@ -16,11 +16,10 @@ Ruby-htslib is the [Ruby](https://www.ruby-lang.org) bindings to [HTSlib](https:
 ## Requirements
 
 - [Ruby](https://github.com/ruby/ruby) 3.1 or above.
-- [HTSlib](https://github.com/samtools/htslib)
+- [HTSlib](https://github.com/samtools/htslib), including headers and `pkg-config` metadata
   - Ubuntu : `apt install libhts-dev`
   - macOS : `brew install htslib`
   - Windows : [mingw-w64-htslib](https://packages.msys2.org/base/mingw-w64-htslib) is automatically fetched when installing the gem ([RubyInstaller](https://rubyinstaller.org) only).
-  - Build from source code (see the Development section)
 
 ## Installation
 
@@ -28,15 +27,15 @@ Ruby-htslib is the [Ruby](https://www.ruby-lang.org) bindings to [HTSlib](https:
 gem install htslib
 ```
 
-If you have installed htslib with apt on Ubuntu or homebrew on Mac, [pkg-config](https://github.com/ruby-gnome/pkg-config)
-will automatically detect the location of the shared library. If pkg-config does not work well, set `PKG_CONFIG_PATH`.
-Alternatively, you can specify the directory of the shared library by setting the environment variable `HTSLIBDIR`.
+The gem builds a native extension and links to the system HTSlib. `pkg-config`
+normally finds it automatically. For a non-standard installation, set
+`PKG_CONFIG_PATH`, pass `--with-htslib-dir`, or set `HTSLIBDIR` while installing.
 
 ```sh
-export HTSLIBDIR="/your/path/to/htslib" # Directory where libhts.so is located
+gem install htslib -- --with-htslib-dir=/your/htslib/prefix
 ```
 
-ruby-htslib also works on Windows. If you use RubyInstaller, htslib will be prepared automatically.
+RubyInstaller installs the declared MSYS2 HTSlib dependency on Windows.
 
 ## Usage
 
@@ -77,6 +76,41 @@ HTS::Bam.open("test/fixtures/moo.bam") do |bam|
 end
 ```
 
+Creating a BAM file from Ruby values
+
+```ruby
+header = HTS::Bam::Header.new
+header.update_hd(version: "1.6", sort_order: "unsorted")
+header.add_sq("chr1", length: 1_000_000)
+
+record = HTS::Bam::Record.new(
+  header,
+  qname: "read1",
+  chrom: "chr1",       # resolved through the header
+  pos: 99,              # zero-based
+  mapq: 60,
+  cigar: "4M",
+  sequence: "ACGT",
+  qualities: [30, 31, 32, 33],
+  aux: { "NM" => 0 }
+)
+
+HTS::Bam.open("output.bam", "wb") do |bam|
+  bam.write_header(header)
+  bam << record
+end
+```
+
+Use `quality_string:` for a Phred+33 string instead of numeric `qualities:`.
+Omit qualities (or pass `nil`) to write missing qualities. `seq:` / `qual:`
+are accepted as concise aliases. Reference names can also be assigned with
+`record.chrom=` and `record.mate_chrom=`.
+
+BAM/CRAM, BCF/VCF, and Tabix indexes are loaded lazily when an indexed
+operation such as `query` or a region-limited pileup first needs them. Pass an
+explicit `index:` path to `open` to validate and load that index immediately.
+`index_loaded?` reports whether the index is currently loaded.
+
 ### HTS::Bcf - VCF / BCF - Variant Call Format file
 
 Reading fields
@@ -88,12 +122,12 @@ bcf = HTS::Bcf.open("test/fixtures/test.bcf")
 
 bcf.each do |r|
   p chrom:  r.chrom,
-    pos:    r.pos,
+    pos:    r.pos + 1, # VCF POS is 1-based; Record#pos is 0-based.
     id:     r.id,
     qual:   r.qual.round(2),
     ref:    r.ref,
     alt:    r.alt,
-    filter: r.filter,
+    filters: r.filter,
     info:   r.info.to_h,
     format: r.format.to_h
 end
@@ -123,35 +157,56 @@ fa.close
 
 ```ruby
 tb = HTS::Tabix.open("test/fixtures/test.vcf.gz")
+# Numeric coordinates are 0-based, half-open: [2000, 3000).
 tb.query("poo", 2000, 3000) do |line|
   puts line.join("\t")
 end
 tb.close
 ```
 
-### Low-level API
+### Low-allocation traversal
 
-Middle architectural layer between high-level Ruby code and low-level C code.
-`HTS::LibHTS` provides native C functions using [Ruby-FFI](https://github.com/ffi/ffi). 
+Large scans can avoid per-sample arrays and genotype strings:
 
 ```ruby
-require 'htslib'
-
-a = HTS::LibHTS.hts_open("a.bam", "r")
-b = HTS::LibHTS.hts_get_format(a)
-p b[:category]
-p b[:format]
+HTS::Bcf.open("variants.bcf", samples: %w[S1 S2], unpack: :format) do |bcf|
+  bcf.each do |record|
+    record.format.each_genotype do |sample_index, genotype|
+      genotype.each_allele do |allele, phased, missing|
+        # Consume the primitive values here.
+      end
+    end
+
+    record.format.each_i32("DP") { |sample_index, depth| }
+    record.format.each_i32_vector("AD") { |sample_index, values| values.each { |depth| } }
+    record.format.each_f32_vector("GL") { |sample_index, values| values.each { |likelihood| } }
+  end
+end
 ```
 
-The low-level API makes it possible to perform detailed operations, such as calling CRAM-specific functions.
+The genotype and numeric vector objects yielded by these iterators are borrowed
+and reused. Consume them inside the block, or call `to_s` / `to_a` to retain an
+owning value. Reading a different FORMAT key does not invalidate a view; reading
+the same key again makes an old view raise `InvalidBorrowedViewError` instead of
+silently returning overwritten data. `genotype_strings`, `get`, and `[]` remain
+the allocating convenience APIs. Use `unpack: :site_only` when FORMAT columns
+are not needed.
 
-#### Macro functions
+Tabix likewise lets callers avoid splitting unused columns. All yielded Ruby
+strings and arrays below are owning values:
 
-HTSlib is designed to improve performance with many macro functions. However, it is not possible to call C macro functions directly from Ruby-FFI. To overcome this, important macro functions have been re-implemented in Ruby, allowing them to be called in the same way as native functions.
+```ruby
+tb.each_line("chr1:1-1000") { |line| }
+tb.each_fields("chr1:1-1000") { |fields| }
+tb.each_selected_fields("chr1:1-1000", 0, 3, 4) { |values| }
+```
 
-#### Garbage Collection and Memory Freeing
+### Native backend migration
 
-A small number of commonly used structs, such as `Bam1` and `Bcf1`, are implemented using FFI's `ManagedStruct`. This allows for automatic memory release when Ruby's garbage collection is triggered. On the other hand, other structs are implemented using `FFI::Struct`, and they will require manual memory release.
+Version 0.5 uses a compiled extension for every supported high-level API. The
+former low-level binding, pointer, and runtime library-switching APIs were
+implementation details and have been removed. Native handles are not exposed;
+install failures report the missing HTSlib build requirement.
 
 ### Need more speed?
 
@@ -165,22 +220,19 @@ Try Crystal. [HTS.cr](https://github.com/bio-cr/hts.cr) is implemented in Crysta
 
 ## Development
 
-#### Compile from source code
+#### Build the extension
 
-[GNU Autotools](https://en.wikipedia.org/wiki/GNU_Autotools) is required to compile htslib.
-To get started with development:
+Install the system HTSlib development package first, then:
 
 ```sh
-git clone --recursive https://github.com/kojix2/ruby-htslib
+git clone https://github.com/kojix2/ruby-htslib
 cd ruby-htslib
 bundle install
-bundle exec rake htslib:build
 bundle exec rake test
 ```
 
-#### Macro functions are reimplemented
-
-HTSlib has many macro functions. These macro functions cannot be called from FFI and must be reimplemented in Ruby.
+Use `bundle exec rake test:local` for fixture-only tests and
+`bundle exec rake test:remote` for the network-dependent URI suite.
 
 #### Use the latest Ruby
 
@@ -196,30 +248,10 @@ Return value
 
 The most challenging part is the return value. In the Crystal language, methods are expected to return only one type. On the other hand, in the Ruby language, methods that return multiple classes are very common. For example, in the Crystal language, the compiler gets confused if the return value is one of six types: Int32, Int64, Float32, Float64, Nil, or String. In fact Crystal allows you to do that. But the code gets a little messy. In Ruby, this is very common and doesn't cause any problems.
 
-Memory management
-
-Ruby and Crystal are languages that use garbage collection. However, the memory release policy for allocated C structures is slightly different: in Ruby-FFI, you can define a `self.release` method in `FFI::Struct`. This method is called when GC. So you don't have to worry about memory in high-level APIs like Bam::Record or Bcf::Record, etc. Crystal requires you to define a finalize method on each class. So you need to define it in Bam::Record or Bcf::Record.
-
-Macro functions
-
-In ruby-htslib, C macro functions are added to `LibHTS`, but in Crystal, `LibHTS` is a Lib, so methods cannot be added. methods are added to `LibHTS2`.
-
 #### Naming convention
 
 If you are not sure about the naming of a method, follow the Rust-htslib API. This is a very weak rule. if a more appropriate name is found later in Ruby, it will replace it.
 
-#### Support for bitfields of structures
-
-Since Ruby-FFI does not support structure bit fields, the following extensions are used.
-
-- [ffi-bitfield](https://github.com/kojix2/ffi-bitfield) - Extension of Ruby-FFI to support bitfields.
-
-#### Automatic validation
-
-In the `script` directory, there are several tools to help implement ruby-htslib. Scripts using c2ffi can check the coverage of htslib functions in Ruby-htslib. They are useful when new versions of htslib are released.
-
-- [c2ffi](https://github.com/rpav/c2ffi) is a tool to create JSON format metadata from C header files.
-
 ## Contributing
 
 Ruby-htslib is a library under development, so even minor improvements like typo fixes are welcome! Please feel free to send us your pull requests.

data/TUTORIAL.md

@@ -17,33 +17,30 @@ Bcf o-- `Bcf::Record`
 `Bcf::Record` o-- Info
 `Bcf::Record` o-- Format
 class Bam{
-  +@hts_file : FFI::Struct
+  -@native
   +@header : Bam::Header
   +@file_name
   +@index_name
   +@mode
-  +struct()
   +build_index()
   +each() Enumerable
   +query()
 }
 class Bcf{
-  +@hts_file : FFI::Struct
+  -@native
   +@header : Bcf::Header
   +@file_name
   +@index_name
   +@mode
-  +struct()
   +build_index()
   +each() Enumerable
   +query()
 }
 class Tabix~Hts~{
-  +@hts_file : FFI::Struct
+  -@native
 }
 class `Bam::Header`{
-  +@sam_hdr : FFI::Struct
-  +struct()
+  -@native
   +target_count()
   +target_names()
   +name2tid()
@@ -51,9 +48,8 @@ class `Bam::Header`{
   +to_s()
 }
 class `Bam::Record` {
-  +@bam1 : FFI::Struct
+  -@native
   +@header : Bam::Header
-  +struct()
   +qname()
   +qname=()
   +tid()
@@ -97,14 +93,12 @@ class `Aux` {
   +get_string()
 }
 class `Bcf::Header`{
-  +@bcf_hdr : FFI::Struct
-  +struct()
+  -@native
   +to_s()
 }
 class `Bcf::Record`{
-  +@bcf1 : FFI::Struct
+  -@native
   +@header : Bcf::Header
-  +struct()
   +rid()
   +rid=()
   +chrom()
@@ -191,13 +185,8 @@ class Faidx{
 gem install htslib
 ```
 
-You can check which shared libraries are used by ruby-htslib as follows
-
-```ruby
-require "htslib"
-puts HTS.lib_path
-# => "/home/kojix2/.rbenv/versions/3.2.0/lib/ruby/gems/3.2.0/gems/htslib-0.2.6/vendor/libhts.so"
-```
+The gem compiles its native extension against the system HTSlib. Install the
+HTSlib development package before installing the gem.
 
 ## HTS::Bam - SAM / BAM / CRAM - Sequence Alignment Map file
 
@@ -342,12 +331,12 @@ bcf = HTS::Bcf.open("b.bcf")
 
 bcf.each do |r|
   p chrom:  r.chrom,
-    pos:    r.pos,
+    pos:    r.pos + 1, # VCF POS is 1-based; Record#pos is 0-based.
     id:     r.id,
     qual:   r.qual.round(2),
     ref:    r.ref,
     alt:    r.alt,
-    filter: r.filter,
+    filters: r.filter,
     info:   r.info.to_h,
     format: r.format.to_h
 end

data/ext/htslib_native/extconf.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require "mkmf"
+
+htslib_dir = with_config("htslib-dir", ENV["HTSLIBDIR"])
+if htslib_dir
+  include_dir = File.directory?(File.join(htslib_dir, "include")) ? File.join(htslib_dir, "include") : htslib_dir
+  library_dir = if File.directory?(File.join(htslib_dir, "lib"))
+                  File.join(htslib_dir, "lib")
+                else
+                  htslib_dir
+                end
+  dir_config("htslib", include_dir, library_dir)
+end
+
+pkg_config("htslib") unless htslib_dir
+
+abort "HTSlib headers were not found. Install the HTSlib development package or set --with-htslib-dir." unless \
+  have_header("htslib/hts.h") && have_header("htslib/sam.h") &&
+  have_header("htslib/vcf.h") && have_header("htslib/faidx.h") && have_header("htslib/tbx.h")
+
+abort "HTSlib library was not found. Install HTSlib or set --with-htslib-dir." unless \
+  have_library("hts", "hts_version")
+
+required_functions = {
+  "sam_read1" => "htslib/sam.h",
+  "bam_plp_init" => "htslib/sam.h",
+  "bam_mplp64_auto" => "htslib/sam.h",
+  "bam_mplp_init_overlaps" => "htslib/sam.h",
+  "hts_base_mod_state_alloc" => "htslib/sam.h",
+  "bcf_read" => "htslib/vcf.h",
+  "bcf_hdr_set" => "htslib/vcf.h",
+  "fai_load3_format" => "htslib/faidx.h",
+  "tbx_index_load3" => "htslib/tbx.h"
+}
+missing_functions = required_functions.reject { |function, header| have_func(function, header) }.keys
+unless missing_functions.empty?
+  abort "HTSlib is missing required functions: #{missing_functions.join(', ')}. Install a supported HTSlib development package."
+end
+
+create_makefile("htslib_native_ext")

data/ext/htslib_native/htslib_native.h

@@ -0,0 +1,11 @@
+#ifndef RUBY_HTSLIB_NATIVE_H
+#define RUBY_HTSLIB_NATIVE_H
+
+#include "ruby.h"
+
+void Init_htslib_native_faidx(VALUE native);
+void Init_htslib_native_tabix(VALUE native);
+void Init_htslib_native_bam(VALUE native);
+void Init_htslib_native_bcf(VALUE native);
+
+#endif

data/ext/htslib_native/htslib_native_ext.c

@@ -0,0 +1,48 @@
+#include "htslib_native.h"
+#include <htslib/hts.h>
+
+static VALUE native_htslib_version(VALUE self)
+{
+    return rb_str_new_cstr(hts_version());
+}
+
+static VALUE selected_fields(VALUE self, VALUE line, VALUE columns)
+{
+    const char *bytes;
+    long length, column_count, field_start = 0, column = 0, offset, i;
+    VALUE result;
+
+    StringValue(line);
+    Check_Type(columns, T_ARRAY);
+    bytes = RSTRING_PTR(line);
+    length = RSTRING_LEN(line);
+    column_count = RARRAY_LEN(columns);
+    result = rb_ary_new_capa(column_count);
+    for (i = 0; i < column_count; i++) rb_ary_push(result, Qnil);
+
+    for (offset = 0; offset <= length; offset++) {
+        if (offset == length || bytes[offset] == '\t') {
+            for (i = 0; i < column_count; i++) {
+                long requested = NUM2LONG(rb_ary_entry(columns, i));
+                if (requested == column) {
+                    rb_ary_store(result, i, rb_str_new(bytes + field_start, offset - field_start));
+                }
+            }
+            column++;
+            field_start = offset + 1;
+        }
+    }
+    return result;
+}
+
+void Init_htslib_native_ext(void)
+{
+    VALUE hts = rb_define_module("HTS");
+    VALUE native = rb_define_module_under(hts, "Native");
+    rb_define_module_function(native, "htslib_version", native_htslib_version, 0);
+    rb_define_module_function(native, "selected_fields", selected_fields, 2);
+    Init_htslib_native_faidx(native);
+    Init_htslib_native_tabix(native);
+    Init_htslib_native_bam(native);
+    Init_htslib_native_bcf(native);
+}