bio-twobit 0.1.3 → 0.2.0

data/lib/bio/twobit/cache_path.rb

@@ -0,0 +1,32 @@
+# https://github.com/red-data-tools/red-datasets/blob/master/lib/datasets/cache-path.rb
+
+module Bio
+  class TwoBit
+    class CachePath
+      def initialize(id)
+        @id = id
+      end
+
+      def base_dir
+        Pathname(system_cache_dir).expand_path + "bio-twobit" + @id
+      end
+
+      def remove
+        FileUtils.rmtree(base_dir.to_s, secure: true) if base_dir.exist?
+      end
+
+      private
+
+      def system_cache_dir
+        case RUBY_PLATFORM
+        when /mswin/, /mingw/
+          ENV["LOCALAPPDATA"] || "~/AppData/Local"
+        when /darwin/
+          "~/Library/Caches"
+        else
+          ENV["XDG_CACHE_HOME"] || "~/.cache"
+        end
+      end
+    end
+  end
+end

data/lib/bio/twobit/downloader.rb

@@ -0,0 +1,282 @@
+# https://github.com/red-data-tools/red-datasets/blob/master/lib/datasets/downloader.rb
+
+require "fileutils"
+begin
+  require "io/console"
+rescue LoadError
+end
+require "net/http"
+require "pathname"
+
+module Bio
+  class TwoBit
+    class Downloader
+      class TooManyRedirects < StandardError; end
+
+      def initialize(url)
+        url = if url.is_a?(URI::Generic)
+                url.dup
+              else
+                URI.parse(url)
+              end
+        @url = url
+        return if @url.is_a?(URI::HTTP)
+
+        raise ArgumentError, "download URL must be HTTP or HTTPS: <#{@url}>"
+      end
+
+      def download(output_path)
+        return if output_path.exist?
+
+        output_path.parent.mkpath
+
+        headers = {
+          "Accept-Encoding" => "identity",
+          "User-Agent" => "BioTwobit/#{VERSION}"
+        }
+        start = nil
+        partial_output_path = Pathname.new("#{output_path}.partial")
+        if partial_output_path.exist?
+          start = partial_output_path.size
+          headers["Range"] = "bytes=#{start}-"
+        end
+
+        start_http(@url, headers) do |response|
+          if response.is_a?(Net::HTTPPartialContent)
+            mode = "ab"
+          else
+            start = nil
+            mode = "wb"
+          end
+
+          base_name = @url.path.split("/").last
+          size_current = 0
+          size_max = response.content_length
+          if start
+            size_current += start
+            size_max += start
+          end
+          progress_reporter = ProgressReporter.new(base_name, size_max)
+          partial_output_path.open(mode) do |output|
+            response.read_body do |chunk|
+              size_current += chunk.bytesize
+              progress_reporter.report(size_current)
+              output.write(chunk)
+            end
+          end
+        end
+        FileUtils.mv(partial_output_path, output_path)
+      rescue TooManyRedirects => e
+        last_url = e.message[/\Atoo many redirections: (.+)\z/, 1]
+        raise TooManyRedirects, "too many redirections: #{@url} .. #{last_url}"
+      end
+
+      private def start_http(url, headers, limit = 10, &block)
+        raise TooManyRedirects, "too many redirections: #{url}" if limit == 0
+
+        http = Net::HTTP.new(url.hostname, url.port)
+        # http.set_debug_output($stderr)
+        http.use_ssl = (url.scheme == "https")
+        http.start do
+          path = url.path
+          path += "?#{url.query}" if url.query
+          request = Net::HTTP::Get.new(path, headers)
+          http.request(request) do |response|
+            case response
+            when Net::HTTPSuccess, Net::HTTPPartialContent
+              return block.call(response)
+            when Net::HTTPRedirection
+              url = URI.parse(response[:location])
+              warn "Redirect to #{url}"
+              return start_http(url, headers, limit - 1, &block)
+            else
+              message = response.code
+              message += ": #{response.message}" if response.message and !response.message.empty?
+              message += ": #{url}"
+              raise response.error_type.new(message, response)
+            end
+          end
+        end
+      end
+
+      class ProgressReporter
+        def initialize(base_name, size_max)
+          @base_name = base_name
+          @size_max = size_max
+
+          @time_previous = Time.now
+          @size_previous = 0
+
+          @need_report = ($stderr == STDERR and $stderr.tty?)
+        end
+
+        def report(size_current)
+          return unless @need_report
+          return if @size_max.nil?
+          return unless foreground?
+
+          done = (size_current == @size_max)
+          time_current = Time.now
+          return if !done and time_current - @time_previous <= 1
+
+          read_bytes = size_current - @size_previous
+          throughput = read_bytes.to_f / (time_current - @time_previous)
+          @time_previous = time_current
+          @size_previous = size_current
+
+          message = build_message(size_current, throughput)
+          $stderr.print("\r#{message}") if message
+          $stderr.puts if done
+        end
+
+        private
+
+        def build_message(size_current, throughput)
+          percent = (size_current / @size_max.to_f) * 100
+          formatted_size = format("[%s/%s]", format_size(size_current), format_size(@size_max))
+          rest_second = (@size_max - size_current) / throughput
+          separator = " - "
+          progress = format("%05.1f%% %s %s %s", percent, formatted_size, format_time_interval(rest_second),
+                            format_throughput(throughput))
+          base_name = @base_name
+
+          width = guess_terminal_width
+          return "#{base_name}#{separator}#{progress}" if width.nil?
+
+          return nil if progress.size > width
+
+          base_name_width = width - progress.size - separator.size
+          if base_name.size > base_name_width
+            ellipsis = "..."
+            shorten_base_name_width = base_name_width - ellipsis.size
+            return progress if shorten_base_name_width < 1
+
+            base_name = base_name[0, shorten_base_name_width] + ellipsis
+
+          end
+          "#{base_name}#{separator}#{progress}"
+        end
+
+        def format_size(size)
+          if size < 1000
+            "%d" % size
+          elsif size < (1000**2)
+            format("%6.2fKB", (size.to_f / 1000))
+          elsif size < (1000**3)
+            format("%6.2fMB", (size.to_f / (1000**2)))
+          elsif size < (1000**4)
+            format("%6.2fGB", (size.to_f / (1000**3)))
+          else
+            format("%.2fTB", (size.to_f / (1000**4)))
+          end
+        end
+
+        def format_time_interval(interval)
+          if interval < 60
+            "00:00:%02d" % interval
+          elsif interval < (60 * 60)
+            minute, second = interval.divmod(60)
+            format("00:%02d:%02d", minute, second)
+          elsif interval < (60 * 60 * 24)
+            minute, second = interval.divmod(60)
+            hour, minute = minute.divmod(60)
+            format("%02d:%02d:%02d", hour, minute, second)
+          else
+            minute, second = interval.divmod(60)
+            hour, minute = minute.divmod(60)
+            day, hour = hour.divmod(24)
+            format("%dd %02d:%02d:%02d", day, hour, minute, second)
+          end
+        end
+
+        def format_throughput(throughput)
+          throughput_byte = throughput / 8
+          if throughput_byte <= 1000
+            "%3dB/s" % throughput_byte
+          elsif throughput_byte <= (1000**2)
+            format("%3dKB/s", (throughput_byte / 1000))
+          elsif throughput_byte <= (1000**3)
+            format("%3dMB/s", (throughput_byte / (1000**2)))
+          else
+            format("%3dGB/s", (throughput_byte / (1000**3)))
+          end
+        end
+
+        def foreground?
+          proc_stat_path = "/proc/self/stat"
+          ps_path = "/bin/ps"
+
+          if File.exist?(proc_stat_path)
+            stat = File.read(proc_stat_path).sub(/\A.+\) /, "").split
+            process_group_id = stat[2]
+            terminal_process_group_id = stat[5]
+            process_group_id == terminal_process_group_id
+          elsif File.executable?(ps_path)
+            IO.pipe do |input, output|
+              pid = spawn(ps_path, "-o", "stat", "-p", Process.pid.to_s,
+                          { out: output, err: output })
+              output.close
+              _, status = Process.waitpid2(pid)
+              return false unless status.success?
+
+              input.each_line.to_a.last.include?("+")
+            end
+          else
+            false
+          end
+        end
+
+        def guess_terminal_width
+          guess_terminal_width_from_io ||
+            guess_terminal_width_from_command ||
+            guess_terminal_width_from_env ||
+            80
+        end
+
+        def guess_terminal_width_from_io
+          if IO.respond_to?(:console)
+            IO.console.winsize[1]
+          elsif $stderr.respond_to?(:winsize)
+            begin
+              $stderr.winsize[1]
+            rescue SystemCallError
+              nil
+            end
+          end
+        end
+
+        def guess_terminal_width_from_command
+          IO.pipe do |input, output|
+            begin
+              pid = spawn("tput", "cols", { out: output, err: output })
+            rescue SystemCallError
+              return nil
+            end
+
+            output.close
+            _, status = Process.waitpid2(pid)
+            return nil unless status.success?
+
+            result = input.read.chomp
+            begin
+              Integer(result, 10)
+            rescue ArgumentError
+              nil
+            end
+          end
+        end
+
+        def guess_terminal_width_from_env
+          env = ENV["COLUMNS"] || ENV["TERM_WIDTH"]
+          return nil if env.nil?
+
+          begin
+            Integer(env, 10)
+          rescue ArgumentError
+            nil
+          end
+        end
+      end
+    end
+  end
+end

data/lib/bio/twobit/metadata.rb

@@ -0,0 +1,9 @@
+module Bio
+  class TwoBit
+    class Metadata < Struct.new(:id,
+                                :name,
+                                :url,
+                                :description)
+    end
+  end
+end

data/lib/bio/twobit/reference_genome.rb

@@ -0,0 +1,24 @@
+module Bio
+  class TwoBit
+    class ReferenceGenome < TwoBit
+      def self.open(*args, **kwargs, &block)
+        raise ArgumentError, "#{self}#open() does not accept arguments" if args.any?
+
+        super(**kwargs, &block)
+      end
+
+      def initialize(**kwargs)
+        @local_name ||= File.basename(@data_url)
+        super(prepare_data, **kwargs)
+      end
+
+      private
+
+      def prepare_data
+        data_path = cache_dir_path + @local_name
+        download(data_path, @data_url)
+        data_path
+      end
+    end
+  end
+end

data/lib/bio/twobit/references/danrer10.rb

@@ -0,0 +1,16 @@
+require_relative "../reference_genome"
+
+module Bio
+  class TwoBit
+    class DanRer10 < ReferenceGenome
+      def initialize(**kwargs)
+        @metadata      = Metadata.new
+        @metadata.id   = "danRer10"
+        @metadata.name = "danRer10"
+        @metadata.url  = "https://hgdownload.soe.ucsc.edu/downloads.html"
+        @data_url      = "https://hgdownload.soe.ucsc.edu/goldenPath/danRer10/bigZips/danRer10.2bit"
+        super(**kwargs)
+      end
+    end
+  end
+end

data/lib/bio/twobit/references/danrer11.rb

@@ -0,0 +1,16 @@
+require_relative "../reference_genome"
+
+module Bio
+  class TwoBit
+    class DanRer11 < ReferenceGenome
+      def initialize(**kwargs)
+        @metadata      = Metadata.new
+        @metadata.id   = "danRer11"
+        @metadata.name = "danRer11"
+        @metadata.url  = "https://hgdownload.soe.ucsc.edu/downloads.html"
+        @data_url      = "https://hgdownload.soe.ucsc.edu/goldenPath/danRer11/bigZips/danRer11.2bit"
+        super(**kwargs)
+      end
+    end
+  end
+end

data/lib/bio/twobit/references/dm6.rb

@@ -0,0 +1,16 @@
+require_relative "../reference_genome"
+
+module Bio
+  class TwoBit
+    class Dm6 < ReferenceGenome
+      def initialize(**kwargs)
+        @metadata      = Metadata.new
+        @metadata.id   = "dm6"
+        @metadata.name = "dm6"
+        @metadata.url  = "https://hgdownload.soe.ucsc.edu/downloads.html"
+        @data_url      = "https://hgdownload.soe.ucsc.edu/goldenPath/dm6/bigZips/dm6.2bit"
+        super(**kwargs)
+      end
+    end
+  end
+end

data/lib/bio/twobit/references/hg19.rb

@@ -0,0 +1,16 @@
+require_relative "../reference_genome"
+
+module Bio
+  class TwoBit
+    class Hg19 < ReferenceGenome
+      def initialize(**kwargs)
+        @metadata      = Metadata.new
+        @metadata.id   = "hg19"
+        @metadata.name = "hg19"
+        @metadata.url  = "https://hgdownload.soe.ucsc.edu/downloads.html"
+        @data_url      = "https://hgdownload.soe.ucsc.edu/goldenPath/hg19/bigZips/hg19.2bit"
+        super(**kwargs)
+      end
+    end
+  end
+end

data/lib/bio/twobit/references/hg38.rb

@@ -0,0 +1,16 @@
+require_relative "../reference_genome"
+
+module Bio
+  class TwoBit
+    class Hg38 < ReferenceGenome
+      def initialize(**kwargs)
+        @metadata      = Metadata.new
+        @metadata.id   = "hg38"
+        @metadata.name = "hg38"
+        @metadata.url  = "https://hgdownload.soe.ucsc.edu/downloads.html"
+        @data_url      = "https://hgdownload.soe.ucsc.edu/goldenPath/hg38/bigZips/hg38.2bit"
+        super(**kwargs)
+      end
+    end
+  end
+end

data/lib/bio/twobit/references/hs1.rb

@@ -0,0 +1,16 @@
+require_relative "../reference_genome"
+
+module Bio
+  class TwoBit
+    class Hs1 < ReferenceGenome
+      def initialize(**kwargs)
+        @metadata      = Metadata.new
+        @metadata.id   = "hs1"
+        @metadata.name = "hs1"
+        @metadata.url  = "https://hgdownload.soe.ucsc.edu/downloads.html"
+        @data_url      = "https://hgdownload.soe.ucsc.edu/goldenPath/hs1/bigZips/hs1.2bit"
+        super(**kwargs)
+      end
+    end
+  end
+end

data/lib/bio/twobit/references/mm10.rb

@@ -0,0 +1,16 @@
+require_relative "../reference_genome"
+
+module Bio
+  class TwoBit
+    class Mm10 < ReferenceGenome
+      def initialize(**kwargs)
+        @metadata      = Metadata.new
+        @metadata.id   = "mm10"
+        @metadata.name = "mm10"
+        @metadata.url  = "https://hgdownload.soe.ucsc.edu/downloads.html"
+        @data_url      = "https://hgdownload.soe.ucsc.edu/goldenPath/mm10/bigZips/mm10.2bit"
+        super(**kwargs)
+      end
+    end
+  end
+end

data/lib/bio/twobit/references/mm39.rb

@@ -0,0 +1,16 @@
+require_relative "../reference_genome"
+
+module Bio
+  class TwoBit
+    class Mm39 < ReferenceGenome
+      def initialize(**kwargs)
+        @metadata      = Metadata.new
+        @metadata.id   = "mm39"
+        @metadata.name = "mm39"
+        @metadata.url  = "https://hgdownload.soe.ucsc.edu/downloads.html"
+        @data_url      = "https://hgdownload.soe.ucsc.edu/goldenPath/mm39/bigZips/mm39.2bit"
+        super(**kwargs)
+      end
+    end
+  end
+end

data/lib/bio/twobit/references/mm9.rb

@@ -0,0 +1,16 @@
+require_relative "../reference_genome"
+
+module Bio
+  class TwoBit
+    class Mm9 < ReferenceGenome
+      def initialize(**kwargs)
+        @metadata      = Metadata.new
+        @metadata.id   = "mm9"
+        @metadata.name = "mm9"
+        @metadata.url  = "https://hgdownload.soe.ucsc.edu/downloads.html"
+        @data_url      = "https://hgdownload.soe.ucsc.edu/goldenPath/mm9/bigZips/mm9.2bit"
+        super(**kwargs)
+      end
+    end
+  end
+end

data/lib/bio/twobit/version.rb

@@ -2,6 +2,6 @@
 
 module Bio
   class TwoBit
-    VERSION = "0.1.3"
+    VERSION = "0.2.0"
   end
 end

data/lib/bio/twobit.rb

@@ -1,26 +1,51 @@
 # frozen_string_literal: true
 
+require "pathname"
 require_relative "twobit/version"
 require_relative "twobit/twobit"
+require_relative "twobit/cache_path"
+require_relative "twobit/downloader"
+require_relative "twobit/metadata"
 
 module Bio
   # Reader for .2bit files (i.e., from UCSC genome browser)
   class TwoBit
-    def self.open(*args, **kwargs)
-      file = new(*args, **kwargs)
-      return file unless block_given?
-
-      begin
-        yield file
-      ensure
-        file.close
+    attr_reader :metadata
+
+    class << self
+      def open(*args, **kwargs)
+        file = new(*args, **kwargs)
+        return file unless block_given?
+
+        begin
+          yield file
+        ensure
+          file.close
+        end
+        file
+      end
+
+      # Here, const_missing is used instead of autoload.
+
+      def const_missing(name)
+        # prevents const_get from being called recursively many times.
+        @missing_const ||= []
+        super if @missing_const.include? name
+        @missing_const << name
+
+        path = File.join(__dir__, "twobit/references", "#{name.to_s.downcase}.rb")
+        if File.exist?(path)
+          require path
+          return const_get(name)
+        end
+        super
       end
-      file
     end
 
     def initialize(fname, masked: false)
       raise "TwoBit::new() does not take block; use TwoBit::open() instead" if block_given?
 
+      fname = fname.to_path if fname.respond_to?(:to_path)
       @fname = fname
       if masked
         mskd = 1
@@ -49,6 +74,8 @@ module Bio
       sequence_raw(chrom, start, stop)
     end
 
+    alias seq sequence
+
     def bases(chrom, start = 0, stop = 0, fraction: true)
       raise ArgumentError, "negative start position" if start.negative?
       raise ArgumentError, "negative stop position" if stop.negative?
@@ -69,5 +96,24 @@ module Bio
 
       soft_masked_blocks_raw(chrom, start, stop)
     end
+
+    def clear_cache!
+      cache_path.remove
+    end
+
+    private
+
+    def cache_dir_path
+      cache_path.base_dir
+    end
+
+    def cache_path
+      @cache_path ||= CachePath.new(@metadata.id)
+    end
+
+    def download(output_path, url)
+      downloader = Downloader.new(url)
+      downloader.download(output_path)
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bio-twobit
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.0
 platform: ruby
 authors:
 - kojix2
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-07-29 00:00:00.000000000 Z
+date: 2023-01-11 00:00:00.000000000 Z
 dependencies: []
 description: This is a Ruby binding for lib2bit(https://github.com/dpryan79/lib2bit),
   which provides high-speed access to genomic data in 2bit file format.
@@ -19,12 +19,8 @@ extensions:
 - ext/bio/twobit/extconf.rb
 extra_rdoc_files: []
 files:
-- ".rubocop.yml"
-- Gemfile
 - LICENSE.txt
 - README.md
-- Rakefile
-- bio-twobit.gemspec
 - ext/bio/twobit/2bit.c
 - ext/bio/twobit/2bit.h
 - ext/bio/twobit/LICENSE
@@ -32,6 +28,19 @@ files:
 - ext/bio/twobit/twobit.c
 - ext/bio/twobit/twobit.h
 - lib/bio/twobit.rb
+- lib/bio/twobit/cache_path.rb
+- lib/bio/twobit/downloader.rb
+- lib/bio/twobit/metadata.rb
+- lib/bio/twobit/reference_genome.rb
+- lib/bio/twobit/references/danrer10.rb
+- lib/bio/twobit/references/danrer11.rb
+- lib/bio/twobit/references/dm6.rb
+- lib/bio/twobit/references/hg19.rb
+- lib/bio/twobit/references/hg38.rb
+- lib/bio/twobit/references/hs1.rb
+- lib/bio/twobit/references/mm10.rb
+- lib/bio/twobit/references/mm39.rb
+- lib/bio/twobit/references/mm9.rb
 - lib/bio/twobit/version.rb
 homepage: https://github.com/ruby-on-bioc/bio-twobit
 licenses:
@@ -52,7 +61,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.4.1
 signing_key: 
 specification_version: 4
 summary: A ruby library for accessing 2bit files

data/.rubocop.yml

@@ -1,13 +0,0 @@
-AllCops:
-  TargetRubyVersion: 2.6
-
-Style/StringLiterals:
-  Enabled: true
-  EnforcedStyle: double_quotes
-
-Style/StringLiteralsInInterpolation:
-  Enabled: true
-  EnforcedStyle: double_quotes
-
-Layout/LineLength:
-  Max: 120