dratools 0.0.1

data/lib/dratools/commands/meta_command.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+require 'json'
+
+require_relative 'base_command'
+
+module Dratools
+  module Commands
+    # DDBJ resource JSON のメタ情報を要約表示する。
+    class MetaCommand < BaseCommand
+      LABEL_WIDTH = 18
+
+      private
+
+      def command_name
+        'meta'
+      end
+
+      def default_options
+        super.merge(json: false)
+      end
+
+      def configure_parser(parser)
+        parser.on('--json', '生の resource JSON を整形して表示する') { @options[:json] = true }
+      end
+
+      def usage_examples
+        [
+          "#{Dratools::NAME} meta DRR300000",
+          "#{Dratools::NAME} meta --json DRR300000"
+        ]
+      end
+
+      def process(accession)
+        record = @resolver.fetch_record_for(accession)
+        if @options[:json]
+          json_buffer << record
+          return
+        end
+
+        print_summary(accession, record)
+      end
+
+      def finalize
+        return unless @options[:json]
+
+        payload = json_buffer.length == 1 ? json_buffer.first : json_buffer
+        @stdout.puts JSON.pretty_generate(payload)
+      end
+
+      def json_buffer
+        @json_buffer ||= []
+      end
+
+      def print_summary(accession, record)
+        summary_pairs(record).each { |label, value| print_field(label, value) }
+        run_count = run_count_for(accession, record)
+        print_field('runs', run_count) if run_count
+      end
+
+      def summary_pairs(record)
+        DdbjRecordFields::INFO_FIELD_KEYS.filter_map do |key|
+          value = normalized_value(record_value(record, key))
+          next if value.nil?
+
+          [field_label(key), value]
+        end
+      end
+
+      def record_value(record, key)
+        return record[key] unless key == DdbjRecordFields::IDENTIFIER_KEY
+
+        record[DdbjRecordFields::IDENTIFIER_KEY] || record[DdbjRecordFields::ACCESSION_KEY]
+      end
+
+      def normalized_value(value)
+        case value
+        when Array
+          values = value.map { |item| normalized_value(item) }.compact
+          values.empty? ? nil : values.join(', ')
+        when Hash
+          compact_hash = value.reject { |_key, item| blank?(item) }
+          compact_hash.empty? ? nil : normalized_hash_value(compact_hash)
+        else
+          normalized_scalar_value(value)
+        end
+      end
+
+      def normalized_scalar_value(value)
+        return nil if blank?(value)
+
+        value.to_s.gsub(/[[:space:]]+/, ' ').strip
+      end
+
+      def normalized_hash_value(value)
+        name = value['name'] || value[:name]
+        blank?(name) ? value.values.join(', ') : name
+      end
+
+      def run_count_for(accession, record)
+        case record[DdbjRecordFields::TYPE_KEY]
+        when DdbjRecordFields::SRA_RUN_RESOURCE_TYPE
+          1
+        when DdbjRecordFields::SRA_EXPERIMENT_RESOURCE_TYPE
+          tree = @resolver.resolve_tree(accession, file_type: @options[:file_type])
+          tree.run_accessions.uniq.length
+        end
+      end
+
+      def blank?(value)
+        value.nil? || (value.respond_to?(:empty?) && value.empty?)
+      end
+
+      def field_label(key)
+        key == DdbjRecordFields::IDENTIFIER_KEY ? 'accession' : key
+      end
+
+      def print_field(label, value)
+        @stdout.puts format("%-#{LABEL_WIDTH}s %s", "#{label}:", value)
+      end
+    end
+  end
+end

data/lib/dratools/commands/probe_command.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require_relative 'base_command'
+
+module Dratools
+  module Commands
+    # 短時間の接続確認だけを行う（完全なダウンロードはしない）。
+    class ProbeCommand < BaseCommand
+      DEFAULT_PROBE_TIMEOUT_SECONDS = DownloadService::DEFAULT_PROBE_TIMEOUT_SECONDS
+      PROBE_SUCCESS_PREFIX = 'OK'
+
+      private
+
+      def command_name
+        'probe'
+      end
+
+      def default_options
+        super.merge(protocol: DEFAULT_PROTOCOL, timeout: DEFAULT_PROBE_TIMEOUT_SECONDS)
+      end
+
+      def configure_parser(parser)
+        add_protocol_option(parser)
+        parser.on('--timeout SEC', Integer,
+                  "接続確認の秒数 (default: #{DEFAULT_PROBE_TIMEOUT_SECONDS})") do |value|
+          @options[:timeout] = value
+        end
+      end
+
+      def usage_examples
+        [
+          "#{Dratools::NAME} probe DRR000001",
+          "#{Dratools::NAME} probe --timeout 10 DRR000001"
+        ]
+      end
+
+      def validate_options
+        super
+        validate_protocol
+        timeout = @options[:timeout]
+        return if timeout.positive?
+
+        raise InvalidOptionError, "invalid --timeout '#{timeout}' (expected: positive integer)"
+      end
+
+      def process(accession)
+        protocol = @options[:protocol]
+        resolve_downloads(accession).each do |download|
+          @downloader.probe_download(download, protocol: protocol, timeout: @options[:timeout])
+          @stdout.puts "#{PROBE_SUCCESS_PREFIX}\t#{download.url_for_protocol(protocol)}"
+        end
+      end
+    end
+  end
+end

data/lib/dratools/commands/runs_command.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+require_relative 'base_command'
+
+module Dratools
+  module Commands
+    # accession を run accession のフラットな一覧に展開する。
+    class RunsCommand < BaseCommand
+      XREF_URL_PATTERN = %r{/(?:resource|search/entry)/sra-run/([^/?#.]+)}
+
+      private
+
+      def command_name
+        'runs'
+      end
+
+      def usage_examples
+        [
+          "#{Dratools::NAME} runs PRJNA341783",
+          "#{Dratools::NAME} runs PRJNA341783 | #{Dratools::NAME} get -O ~/Downloads"
+        ]
+      end
+
+      def process(accession)
+        direct_runs = direct_run_accessions_for(accession)
+        if direct_runs.any?
+          run_accessions.concat(direct_runs)
+          return
+        end
+
+        tree = @resolver.resolve_tree(accession, file_type: @options[:file_type])
+        run_accessions.concat(tree.run_accessions)
+      end
+
+      def finalize
+        run_accessions.uniq.each { |run_accession| @stdout.puts run_accession }
+      end
+
+      def run_accessions
+        @run_accessions ||= []
+      end
+
+      def direct_run_accessions_for(accession)
+        record = @resolver.fetch_record_for(accession)
+        if record[DdbjRecordFields::TYPE_KEY] == DdbjRecordFields::SRA_RUN_RESOURCE_TYPE
+          return [record_accession(record)].compact
+        end
+
+        record.fetch(DdbjRecordFields::DB_XREFS_KEY, []).filter_map do |xref|
+          next unless xref[DdbjRecordFields::TYPE_KEY] == DdbjRecordFields::SRA_RUN_RESOURCE_TYPE
+
+          xref[DdbjRecordFields::IDENTIFIER_KEY] ||
+            xref[DdbjRecordFields::ID_KEY] ||
+            run_accession_from_url(xref[DdbjRecordFields::URL_KEY])
+        end
+      end
+
+      def record_accession(record)
+        record[DdbjRecordFields::ACCESSION_KEY] ||
+          record[DdbjRecordFields::IDENTIFIER_KEY] ||
+          record[DdbjRecordFields::ID_KEY] ||
+          record[DdbjRecordFields::PRIMARY_ID_KEY]
+      end
+
+      def run_accession_from_url(url)
+        url.to_s.match(XREF_URL_PATTERN)&.[](1)
+      end
+    end
+  end
+end

data/lib/dratools/commands/size_command.rb

@@ -0,0 +1,163 @@
+# frozen_string_literal: true
+
+require 'json'
+
+require_relative '../byte_formatter'
+require_relative '../config'
+require_relative '../ddbj_record_fields'
+require_relative 'base_command'
+
+module Dratools
+  module Commands
+    # 実ファイルの Content-Length を HEAD で集計する。
+    class SizeCommand < BaseCommand
+      DEFAULT_SIZE_TIMEOUT_SECONDS = DownloadService::DEFAULT_SIZE_TIMEOUT_SECONDS
+      TSV_COLUMNS = %w[accession files size unresolved].freeze
+
+      private
+
+      def command_name
+        'size'
+      end
+
+      def default_options
+        super.merge(
+          protocol: DEFAULT_PROTOCOL,
+          timeout: DEFAULT_SIZE_TIMEOUT_SECONDS,
+          bytes: false,
+          json: false,
+          per_run: false
+        )
+      end
+
+      def configure_parser(parser)
+        add_protocol_option(parser)
+        parser.on('--timeout SEC', Integer,
+                  "HEAD/ディレクトリ取得の秒数 (default: #{DEFAULT_SIZE_TIMEOUT_SECONDS})") do |value|
+          @options[:timeout] = value
+        end
+        parser.on('--bytes', 'サイズをバイト数で表示する') { @options[:bytes] = true }
+        parser.on('-r', '--per-run', '親 accession を run accession ごとに分けて集計する') do
+          @options[:per_run] = true
+        end
+        parser.on('--json', 'サイズ集計を JSON で表示する') { @options[:json] = true }
+      end
+
+      def usage_examples
+        [
+          "#{Dratools::NAME} size DRR000001",
+          "#{Dratools::NAME} size --type fastq PRJNA341783",
+          "#{Dratools::NAME} size --per-run DRX000001",
+          "#{Dratools::NAME} size --json DRR000001 DRR000002"
+        ]
+      end
+
+      def validate_options
+        super
+        validate_protocol
+        timeout = @options[:timeout]
+        return if timeout.positive?
+
+        raise InvalidOptionError, "invalid --timeout '#{timeout}' (expected: positive integer)"
+      end
+
+      def process(accession)
+        ddbj_record = fetch_record_for_size(accession)
+        downloads = @resolver.resolve_downloads_from_record(
+          accession,
+          ddbj_record,
+          file_type: @options[:file_type]
+        )
+        if @options[:per_run]
+          downloads.group_by(&:run_accession).each do |run_accession, group|
+            record_result(run_accession, group)
+          end
+        else
+          record_result(accession, downloads)
+        end
+      end
+
+      def record_result(label, downloads)
+        lengths = downloads.flat_map do |download|
+          @downloader.content_lengths(
+            download,
+            protocol: @options[:protocol],
+            timeout: @options[:timeout]
+          )
+        end
+        result = {
+          accession: label,
+          file_count: lengths.length,
+          total_size: lengths.compact.sum,
+          unresolved_count: lengths.count(&:nil?)
+        }
+        results << result
+        print_text_result(result) unless @options[:json]
+      end
+
+      def finalize
+        if @options[:json]
+          @stdout.puts JSON.pretty_generate(results)
+          return
+        end
+
+        return if results.length < 2
+
+        # --per-run の集計行はデータ行と混ぜず標準エラーに出し、stdout を純粋な TSV に保つ。
+        print_text_result(total_result, io: @options[:per_run] ? @stderr : @stdout)
+      end
+
+      def results
+        @results ||= []
+      end
+
+      def total_result
+        {
+          accession: 'total',
+          file_count: results.sum { |result| result[:file_count] },
+          total_size: results.sum { |result| result[:total_size] },
+          unresolved_count: results.sum { |result| result[:unresolved_count] }
+        }
+      end
+
+      def print_text_result(result, io: @stdout)
+        emit_tsv_header(TSV_COLUMNS)
+        io.puts [
+          result[:accession],
+          result[:file_count],
+          formatted_size(result),
+          result[:unresolved_count]
+        ].join("\t")
+      end
+
+      def formatted_size(result)
+        size = result[:total_size]
+        return MISSING_VALUE if size.zero? && result[:unresolved_count].positive?
+        return size.to_s if @options[:bytes]
+
+        ByteFormatter.format(size)
+      end
+
+      def fetch_record_for_size(accession)
+        max_direct_runs = Config.size_max_direct_runs
+        ddbj_record = @resolver.fetch_record_for(accession)
+        validate_direct_run_expansion_size!(accession, ddbj_record, max_direct_runs)
+        ddbj_record
+      end
+
+      def validate_direct_run_expansion_size!(accession, ddbj_record, max_direct_runs)
+        return unless max_direct_runs
+
+        direct_run_count = ddbj_record.fetch(DdbjRecordFields::DB_XREFS_KEY, []).count do |xref|
+          xref[DdbjRecordFields::TYPE_KEY] == DdbjRecordFields::SRA_RUN_RESOURCE_TYPE
+        end
+        return if direct_run_count <= max_direct_runs
+
+        raise InvalidRecordError,
+              "#{accession.to_s.upcase} has #{direct_run_count} direct runs; " \
+              "size expands at most #{max_direct_runs} direct runs from one parent accession. " \
+              "Use `#{Dratools::NAME} runs #{accession}` and pass narrower accessions."
+      end
+    end
+  end
+end

data/lib/dratools/commands/tree_command.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require_relative 'base_command'
+require_relative '../config'
+require_relative '../tree_renderer'
+
+module Dratools
+  module Commands
+    # accession から run へ辿る探索ツリーを表示する。
+    class TreeCommand < BaseCommand
+      private
+
+      def command_name
+        'tree'
+      end
+
+      def usage_examples
+        [
+          "#{Dratools::NAME} tree PRJNA341783",
+          "#{Dratools::NAME} tree --type fastq PRJNA341783"
+        ]
+      end
+
+      def process(accession)
+        direct_run_fetch_limit = Config.tree_max_direct_runs
+        tree = @resolver.resolve_tree(
+          accession,
+          file_type: @options[:file_type],
+          direct_run_fetch_limit: direct_run_fetch_limit
+        )
+        @stdout.puts TreeRenderer.new(
+          file_type: @options[:file_type],
+          summary_threshold: summary_threshold(direct_run_fetch_limit)
+        ).render(tree)
+      end
+
+      def summary_threshold(direct_run_fetch_limit)
+        [
+          TreeRenderer::DEFAULT_SUMMARY_THRESHOLD,
+          direct_run_fetch_limit
+        ].compact.min
+      end
+    end
+  end
+end

data/lib/dratools/commands/url_command.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+require 'json'
+
+require_relative '../config'
+require_relative '../ddbj_record_fields'
+require_relative 'base_command'
+
+module Dratools
+  module Commands
+    # ダウンロード URL を表示する（テキストまたは JSON）。
+    class UrlCommand < BaseCommand
+      TSV_COLUMNS = %w[run_accession type url size md5].freeze
+
+      private
+
+      def command_name
+        'url'
+      end
+
+      def default_options
+        super.merge(protocol: DEFAULT_PROTOCOL, json: false, tsv: false)
+      end
+
+      def configure_parser(parser)
+        add_protocol_option(parser)
+        parser.on('--tsv', 'run_accession, type, url, size, md5 を TAB 区切りで表示する') do
+          @options[:tsv] = true
+        end
+        parser.on('--json', 'URL 情報を JSON で表示する') { @options[:json] = true }
+      end
+
+      def usage_examples
+        [
+          "#{Dratools::NAME} url DRR000001",
+          "#{Dratools::NAME} url --protocol ftp DRR000001",
+          "#{Dratools::NAME} url --tsv DRR000001 | grep -v '^#' | cut -f3",
+          "#{Dratools::NAME} url --json DRR000001 DRR000002"
+        ]
+      end
+
+      def validate_options
+        super
+        validate_protocol
+      end
+
+      def process(accession)
+        ddbj_record = fetch_record_for_url(accession)
+        downloads = @resolver.resolve_downloads_from_record(
+          accession,
+          ddbj_record,
+          file_type: @options[:file_type]
+        )
+        if @options[:json]
+          json_buffer.concat(downloads.map { |download| download_to_hash(download) })
+        elsif @options[:tsv]
+          emit_tsv_header(TSV_COLUMNS)
+          downloads.each { |download| @stdout.puts tsv_row(download) }
+        else
+          downloads.each { |download| @stdout.puts download.url_for_protocol(@options[:protocol]) }
+        end
+      end
+
+      def finalize
+        return unless @options[:json]
+
+        @stdout.puts JSON.pretty_generate(json_buffer)
+      end
+
+      def json_buffer
+        @json_buffer ||= []
+      end
+
+      def fetch_record_for_url(accession)
+        max_direct_runs = Config.url_max_direct_runs
+        ddbj_record = @resolver.fetch_record_for(accession)
+        validate_direct_run_expansion_size!(accession, ddbj_record, max_direct_runs)
+        ddbj_record
+      end
+
+      def validate_direct_run_expansion_size!(accession, ddbj_record, max_direct_runs)
+        return unless max_direct_runs
+
+        direct_run_count = ddbj_record.fetch(DdbjRecordFields::DB_XREFS_KEY, []).count do |xref|
+          xref[DdbjRecordFields::TYPE_KEY] == DdbjRecordFields::SRA_RUN_RESOURCE_TYPE
+        end
+        return if direct_run_count <= max_direct_runs
+
+        raise InvalidRecordError,
+              "#{accession.to_s.upcase} has #{direct_run_count} direct runs; " \
+              "url expands at most #{max_direct_runs} direct runs from one parent accession. " \
+              "Use `#{Dratools::NAME} runs #{accession}` and pass narrower accessions, " \
+              "or set #{Config::URL_MAX_DIRECT_RUNS_ENV}=unlimited."
+      end
+
+      def tsv_row(download)
+        [
+          download.run_accession,
+          download.type,
+          download.url_for_protocol(@options[:protocol]),
+          download.size || MISSING_VALUE,
+          download.md5 || MISSING_VALUE
+        ].join("\t")
+      end
+
+      def download_to_hash(download)
+        {
+          run_accession: download.run_accession,
+          type: download.type,
+          url: download.url,
+          ftp_url: download.ftp_url,
+          size: download.size,
+          md5: download.md5
+        }
+      end
+    end
+  end
+end

data/lib/dratools/config.rb

@@ -0,0 +1,114 @@
+# frozen_string_literal: true
+
+require_relative 'errors'
+
+module Dratools
+  # Reads advanced configuration from environment variables.
+  module Config
+    MAX_RECURSIVE_NON_RUN_XREFS_ENV = 'DRATOOLS_MAX_RECURSIVE_NON_RUN_XREFS'
+    TREE_MAX_DIRECT_RUNS_ENV = 'DRATOOLS_TREE_MAX_DIRECT_RUNS'
+    URL_MAX_DIRECT_RUNS_ENV = 'DRATOOLS_URL_MAX_DIRECT_RUNS'
+    SIZE_MAX_DIRECT_RUNS_ENV = 'DRATOOLS_SIZE_MAX_DIRECT_RUNS'
+    DOWNLOAD_CONNECT_TIMEOUT_ENV = 'DRATOOLS_DOWNLOAD_CONNECT_TIMEOUT'
+    DOWNLOAD_STALL_TIMEOUT_ENV = 'DRATOOLS_DOWNLOAD_STALL_TIMEOUT'
+    DOWNLOAD_STALL_SPEED_ENV = 'DRATOOLS_DOWNLOAD_STALL_SPEED'
+    DOWNLOAD_RETRY_COUNT_ENV = 'DRATOOLS_DOWNLOAD_RETRY_COUNT'
+    DOWNLOAD_RETRY_WAIT_ENV = 'DRATOOLS_DOWNLOAD_RETRY_WAIT'
+
+    DEFAULT_MAX_RECURSIVE_NON_RUN_XREFS = 100
+    DEFAULT_TREE_MAX_DIRECT_RUNS = 50
+    DEFAULT_URL_MAX_DIRECT_RUNS = 50
+    DEFAULT_SIZE_MAX_DIRECT_RUNS = 50
+    DEFAULT_DOWNLOAD_CONNECT_TIMEOUT_SECONDS = 30
+    DEFAULT_DOWNLOAD_STALL_TIMEOUT_SECONDS = 60
+    DEFAULT_DOWNLOAD_STALL_SPEED_BYTES_PER_SECOND = 1024
+    DEFAULT_DOWNLOAD_RETRY_COUNT = 3
+    DEFAULT_DOWNLOAD_RETRY_WAIT_SECONDS = 5
+    UNLIMITED_VALUE = 'unlimited'
+
+    module_function
+
+    def max_recursive_non_run_xrefs
+      positive_integer_or_unlimited(
+        MAX_RECURSIVE_NON_RUN_XREFS_ENV,
+        DEFAULT_MAX_RECURSIVE_NON_RUN_XREFS
+      )
+    end
+
+    def tree_max_direct_runs
+      positive_integer_or_unlimited(
+        TREE_MAX_DIRECT_RUNS_ENV,
+        DEFAULT_TREE_MAX_DIRECT_RUNS
+      )
+    end
+
+    def url_max_direct_runs
+      positive_integer_or_unlimited(URL_MAX_DIRECT_RUNS_ENV, DEFAULT_URL_MAX_DIRECT_RUNS)
+    end
+
+    def size_max_direct_runs
+      positive_integer_or_unlimited(SIZE_MAX_DIRECT_RUNS_ENV, DEFAULT_SIZE_MAX_DIRECT_RUNS)
+    end
+
+    def download_connect_timeout_seconds
+      positive_integer(DOWNLOAD_CONNECT_TIMEOUT_ENV, DEFAULT_DOWNLOAD_CONNECT_TIMEOUT_SECONDS)
+    end
+
+    def download_stall_timeout_seconds
+      positive_integer(DOWNLOAD_STALL_TIMEOUT_ENV, DEFAULT_DOWNLOAD_STALL_TIMEOUT_SECONDS)
+    end
+
+    def download_stall_speed_bytes_per_second
+      positive_integer(DOWNLOAD_STALL_SPEED_ENV, DEFAULT_DOWNLOAD_STALL_SPEED_BYTES_PER_SECOND)
+    end
+
+    def download_retry_count
+      non_negative_integer(DOWNLOAD_RETRY_COUNT_ENV, DEFAULT_DOWNLOAD_RETRY_COUNT)
+    end
+
+    def download_retry_wait_seconds
+      positive_integer(DOWNLOAD_RETRY_WAIT_ENV, DEFAULT_DOWNLOAD_RETRY_WAIT_SECONDS)
+    end
+
+    def positive_integer_or_unlimited(name, default)
+      value = ENV.fetch(name, '').strip
+      return default if value.empty?
+      return nil if value.casecmp?(UNLIMITED_VALUE)
+
+      integer = Integer(value, 10)
+      return integer if integer.positive?
+
+      invalid_environment_value!(name, value, "positive integer or #{UNLIMITED_VALUE}")
+    rescue ArgumentError
+      invalid_environment_value!(name, value, "positive integer or #{UNLIMITED_VALUE}")
+    end
+
+    def positive_integer(name, default)
+      value = ENV.fetch(name, '').strip
+      return default if value.empty?
+
+      integer = Integer(value, 10)
+      return integer if integer.positive?
+
+      invalid_environment_value!(name, value, 'positive integer')
+    rescue ArgumentError
+      invalid_environment_value!(name, value, 'positive integer')
+    end
+
+    def non_negative_integer(name, default)
+      value = ENV.fetch(name, '').strip
+      return default if value.empty?
+
+      integer = Integer(value, 10)
+      return integer unless integer.negative?
+
+      invalid_environment_value!(name, value, 'non-negative integer')
+    rescue ArgumentError
+      invalid_environment_value!(name, value, 'non-negative integer')
+    end
+
+    def invalid_environment_value!(name, value, expected)
+      raise InvalidOptionError, "invalid #{name} '#{value}' (expected: #{expected})"
+    end
+  end
+end