dratools 0.0.1

data/lib/dratools/accession_input_collector.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require_relative 'errors'
+
+module Dratools
+  # 引数・ファイル・標準入力からアクセッションを集める。
+  class AccessionInputCollector
+    STANDARD_INPUT_PATH = '-'
+    MISSING_ACCESSION_ARGUMENT = 'ACCESSION'
+    INPUT_OPTION_NAME = '--input'
+
+    def initialize(argv:, input_path: nil, stdin: $stdin)
+      @argv = argv
+      @input_path = input_path
+      @stdin = stdin
+    end
+
+    def collect_accessions
+      accessions = (positional_accessions + streamed_accessions).uniq
+      raise MissingAccessionError, "#{MISSING_ACCESSION_ARGUMENT} is required" if accessions.empty?
+
+      accessions
+    end
+
+    private
+
+    def positional_accessions
+      @argv.map { |value| normalize_accession(value) }.reject(&:empty?)
+    end
+
+    def streamed_accessions
+      return parse_accessions(@stdin.read) if @input_path == STANDARD_INPUT_PATH
+      return parse_accessions(File.read(@input_path)) if @input_path
+      return [] if stdin_tty?
+
+      parse_accessions(@stdin.read)
+    rescue SystemCallError => error
+      raise InputFileError, "#{INPUT_OPTION_NAME}: #{error.message}"
+    end
+
+    def parse_accessions(content)
+      content.each_line.map { |value| normalize_accession(value) }.reject(&:empty?)
+    end
+
+    def normalize_accession(value)
+      value.to_s.strip.upcase
+    end
+
+    def stdin_tty?
+      @stdin.respond_to?(:tty?) && @stdin.tty?
+    end
+  end
+end

data/lib/dratools/accession_resolver.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require_relative 'accession_resource_type_classifier'
+require_relative 'ddbj_record_fields'
+require_relative 'ddbj_resource_client'
+require_relative 'download_candidate_builder'
+require_relative 'errors'
+require_relative 'run_record_collector'
+require_relative 'traversal_node'
+
+module Dratools
+  # accession を受け取り、DDBJ 上の実ファイル候補へ解決する調停役。
+  class AccessionResolver
+    FILE_TYPE_SRA = DdbjRecordFields::FILE_TYPE_SRA
+    FILE_TYPE_FASTQ = DdbjRecordFields::FILE_TYPE_FASTQ
+    FILE_TYPE_ALL = DdbjRecordFields::FILE_TYPE_ALL
+
+    def initialize(
+      client: DdbjResourceClient.new,
+      resource_type_classifier: AccessionResourceTypeClassifier.new,
+      run_record_collector: nil,
+      download_candidate_builder: DownloadCandidateBuilder.new
+    )
+      @client = client
+      @resource_type_classifier = resource_type_classifier
+      @run_record_collector = run_record_collector || RunRecordCollector.new(client: client)
+      @download_candidate_builder = download_candidate_builder
+    end
+
+    def resolve_downloads(accession, file_type: FILE_TYPE_SRA)
+      accession = accession.to_s.upcase
+      ddbj_record = fetch_record_for(accession)
+      resolve_downloads_from_record(accession, ddbj_record, file_type: file_type)
+    end
+
+    def resolve_downloads_from_record(accession, ddbj_record, file_type: FILE_TYPE_SRA)
+      tree = resolve_tree_from_record(accession, ddbj_record, file_type: file_type, tolerant: false)
+      downloads = tree.downloads
+      if downloads.empty?
+        raise NotFoundError, "download URL not found: #{accession.to_s.upcase} (type=#{file_type})"
+      end
+
+      downloads
+    end
+
+    def resolve_tree(accession, file_type: FILE_TYPE_SRA, tolerant: true,
+                     direct_run_fetch_limit: nil)
+      accession = accession.to_s.upcase
+      ddbj_record = fetch_record_for(accession)
+      resolve_tree_from_record(
+        accession,
+        ddbj_record,
+        file_type: file_type,
+        tolerant: tolerant,
+        direct_run_fetch_limit: direct_run_fetch_limit
+      )
+    end
+
+    def resolve_tree_from_record(_accession, ddbj_record, file_type: FILE_TYPE_SRA, tolerant: true,
+                                 direct_run_fetch_limit: nil)
+      tree = @run_record_collector.explore(
+        ddbj_record,
+        tolerant: tolerant,
+        direct_run_fetch_limit: direct_run_fetch_limit
+      )
+      attach_downloads(tree, file_type: file_type)
+      tree
+    end
+
+    def fetch_record_for(accession)
+      resource_type = resource_type_for(accession)
+      @client.fetch_resource_record(resource_type, accession)
+    end
+
+    def resource_type_for(accession)
+      @resource_type_classifier.resource_type_for(accession)
+    end
+
+    private
+
+    def attach_downloads(node, file_type:)
+      if node.run? && node.record
+        downloads = @download_candidate_builder.build_from_run_record(node.record)
+        downloads.select! { |download| file_type == FILE_TYPE_ALL || download.type == file_type }
+        node.children.concat(downloads.map { |download| download_node(download) })
+      end
+
+      node.children.each do |child|
+        attach_downloads(child, file_type: file_type) unless child.download?
+      end
+      node
+    end
+
+    def download_node(download)
+      TraversalNode.new(
+        relation: TraversalNode::DOWNLOAD_RELATION,
+        type: download.type,
+        accession: download.run_accession,
+        url: download.url,
+        download: download
+      )
+    end
+  end
+end

data/lib/dratools/accession_resource_type_classifier.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require_relative 'ddbj_record_fields'
+require_relative 'errors'
+
+module Dratools
+  # accession の接頭辞から DDBJ resource API の type を判定する。
+  class AccessionResourceTypeClassifier
+    RUN_PREFIXES = /\A[DES]RR\d+\z/
+    EXPERIMENT_PREFIXES = /\A[DES]RX\d+\z/
+    SAMPLE_PREFIXES = /\A[DES]RS\d+\z/
+    STUDY_PREFIXES = /\A[DES]RP\d+\z/
+    SUBMISSION_PREFIXES = /\A[DES]RA\d+\z/
+    BIOPROJECT_PREFIXES = /\APRJ(?:DA|DB|NA|EB)\d+\z/
+    BIOSAMPLE_PREFIXES = /\ASAM(?:D|N|EA|EG)?\d+\z/
+
+    TYPE_BY_ACCESSION = [
+      [RUN_PREFIXES, DdbjRecordFields::SRA_RUN_RESOURCE_TYPE],
+      [EXPERIMENT_PREFIXES, DdbjRecordFields::SRA_EXPERIMENT_RESOURCE_TYPE],
+      [SAMPLE_PREFIXES, DdbjRecordFields::SRA_SAMPLE_RESOURCE_TYPE],
+      [STUDY_PREFIXES, DdbjRecordFields::SRA_STUDY_RESOURCE_TYPE],
+      [SUBMISSION_PREFIXES, DdbjRecordFields::SRA_SUBMISSION_RESOURCE_TYPE],
+      [BIOPROJECT_PREFIXES, DdbjRecordFields::BIOPROJECT_RESOURCE_TYPE],
+      [BIOSAMPLE_PREFIXES, DdbjRecordFields::BIOSAMPLE_RESOURCE_TYPE]
+    ].freeze
+
+    def resource_type_for(accession)
+      matching_rule = TYPE_BY_ACCESSION.find { |pattern, _resource_type| pattern.match?(accession) }
+      raise UnsupportedAccessionError, "unsupported accession: #{accession}" unless matching_rule
+
+      matching_rule.last
+    end
+  end
+end

data/lib/dratools/byte_formatter.rb

@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Dratools
+  # Byte count formatter for human-readable IEC units.
+  module ByteFormatter
+    UNITS = %w[B KiB MiB GiB TiB PiB].freeze
+    UNIT_BASE = 1024.0
+
+    module_function
+
+    def format(bytes)
+      value = bytes.to_f
+      unit_index = 0
+
+      while value >= UNIT_BASE && unit_index < UNITS.length - 1
+        value /= UNIT_BASE
+        unit_index += 1
+      end
+
+      return "#{bytes.to_i} B" if unit_index.zero?
+
+      "#{Kernel.format('%.1f', value)} #{UNITS[unit_index]}"
+    end
+  end
+end

data/lib/dratools/checksum_verifier.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require 'digest/md5'
+
+require_relative 'errors'
+
+module Dratools
+  # ダウンロード済みファイルのチェックサムを検証する。
+  #
+  # Digest::MD5.file はストリーム処理なので、巨大ファイルでも全体をメモリに載せない。
+  class ChecksumVerifier
+    def md5_matches?(path, expected_md5)
+      md5_for(path).casecmp?(normalize_md5(expected_md5))
+    end
+
+    def verify_md5!(path, expected_md5)
+      expected_md5 = normalize_md5(expected_md5)
+      actual_md5 = md5_for(path)
+      return true if actual_md5.casecmp?(expected_md5)
+
+      raise ChecksumError, "MD5 mismatch for #{path}: expected #{expected_md5}, got #{actual_md5}"
+    end
+
+    private
+
+    def normalize_md5(md5)
+      md5.to_s.strip
+    end
+
+    def md5_for(path)
+      Digest::MD5.file(path).hexdigest
+    end
+  end
+end

data/lib/dratools/command_line_interface.rb

@@ -0,0 +1,138 @@
+# frozen_string_literal: true
+
+require_relative 'version'
+require_relative 'accession_resolver'
+require_relative 'download_service'
+require_relative 'commands/url_command'
+require_relative 'commands/get_command'
+require_relative 'commands/probe_command'
+require_relative 'commands/tree_command'
+require_relative 'commands/meta_command'
+require_relative 'commands/runs_command'
+require_relative 'commands/size_command'
+
+module Dratools
+  # サブコマンドを振り分ける CLI の入口。
+  class CommandLineInterface
+    COMMAND_NAME = Dratools::NAME
+    SUCCESS_EXIT_STATUS = 0
+    FAILURE_EXIT_STATUS = 1
+
+    SUBCOMMANDS = {
+      'url' => Commands::UrlCommand,
+      'get' => Commands::GetCommand,
+      'probe' => Commands::ProbeCommand,
+      'tree' => Commands::TreeCommand,
+      'meta' => Commands::MetaCommand,
+      'runs' => Commands::RunsCommand,
+      'size' => Commands::SizeCommand
+    }.freeze
+
+    # 単複の打ち間違いを救う別名。左を打っても右の正規コマンドが動く。
+    # ヘルプやエラー・バナーには常に正規名（右）を表示する。
+    SUBCOMMAND_ALIASES = {
+      'run' => 'runs',
+      'urls' => 'url',
+      'sizes' => 'size',
+      'trees' => 'tree'
+    }.freeze
+
+    HELP_FLAGS = ['-h', '--help', 'help'].freeze
+    VERSION_FLAGS = ['-v', '--version', 'version'].freeze
+
+    SUBCOMMAND_SUMMARIES = {
+      'url' => 'ダウンロード URL を表示する (--json で JSON)',
+      'get' => 'ファイルをダウンロードする',
+      'probe' => '短時間の接続確認だけ行う',
+      'tree' => '探索ツリーを表示する',
+      'meta' => 'レコードのメタ情報を表示する (--json で生 JSON)',
+      'runs' => 'run accession の一覧を出力する',
+      'size' => 'ダウンロード合計サイズを集計する'
+    }.freeze
+
+    USAGE_EXAMPLES = [
+      "#{COMMAND_NAME} url DRR000001",
+      "#{COMMAND_NAME} meta DRR000001",
+      "#{COMMAND_NAME} runs PRJNA341783",
+      "#{COMMAND_NAME} size PRJNA341783",
+      "#{COMMAND_NAME} get -O ~/Downloads DRR000001",
+      "#{COMMAND_NAME} tree PRJNA341783",
+      "#{COMMAND_NAME} url --input accessions.txt",
+      "printf 'DRR000001\\nDRR000002\\n' | #{COMMAND_NAME} url"
+    ].freeze
+
+    def self.start(argv)
+      new(argv).run
+    end
+
+    def initialize(
+      argv,
+      resolver: AccessionResolver.new,
+      downloader: DownloadService.new,
+      stdout: $stdout,
+      stderr: $stderr,
+      stdin: $stdin
+    )
+      @argv = argv
+      @resolver = resolver
+      @downloader = downloader
+      @stdout = stdout
+      @stderr = stderr
+      @stdin = stdin
+    end
+
+    def run
+      name = @argv.first
+
+      if name.nil?
+        print_help(@stderr)
+        return FAILURE_EXIT_STATUS
+      end
+      if HELP_FLAGS.include?(name)
+        print_help(@stdout)
+        return SUCCESS_EXIT_STATUS
+      end
+      if VERSION_FLAGS.include?(name)
+        @stdout.puts VERSION
+        return SUCCESS_EXIT_STATUS
+      end
+
+      command_class = SUBCOMMANDS[name] || SUBCOMMANDS[SUBCOMMAND_ALIASES[name]]
+      unless command_class
+        expected = SUBCOMMANDS.keys.join(', ')
+        @stderr.puts "#{COMMAND_NAME}: unknown command '#{name}' (expected: #{expected})"
+        return FAILURE_EXIT_STATUS
+      end
+
+      command_class.new(
+        @argv.drop(1),
+        resolver: @resolver,
+        downloader: @downloader,
+        stdout: @stdout,
+        stderr: @stderr,
+        stdin: @stdin
+      ).run
+    end
+
+    private
+
+    def print_help(stream)
+      stream.puts "Usage: #{COMMAND_NAME} <command> [options] [ACCESSION ...]"
+      stream.puts ''
+      stream.puts 'Commands:'
+      SUBCOMMAND_SUMMARIES.each do |name, summary|
+        stream.puts format('  %-7<name>s %<summary>s', name: name, summary: summary)
+      end
+      stream.puts ''
+      stream.puts 'Aliases:'
+      SUBCOMMAND_ALIASES.each do |alias_name, canonical|
+        stream.puts format('  %-7<a>s -> %<c>s', a: alias_name, c: canonical)
+      end
+      stream.puts ''
+      stream.puts "Run '#{COMMAND_NAME} <command> --help' for command options."
+      stream.puts ''
+      stream.puts 'Examples:'
+      USAGE_EXAMPLES.each { |example| stream.puts "  #{example}" }
+    end
+  end
+end

data/lib/dratools/commands/base_command.rb

@@ -0,0 +1,189 @@
+# frozen_string_literal: true
+
+require 'optparse'
+
+require_relative '../errors'
+require_relative '../download_candidate'
+require_relative '../accession_resolver'
+require_relative '../download_service'
+require_relative '../accession_input_collector'
+
+module Dratools
+  module Commands
+    # サブコマンド共通の土台。オプション解析・accession 収集・例外処理・終了コードを担う。
+    class BaseCommand
+      SUCCESS_EXIT_STATUS = 0
+      FAILURE_EXIT_STATUS = 1
+
+      DEFAULT_FILE_TYPE = AccessionResolver::FILE_TYPE_SRA
+      VALID_FILE_TYPES = [
+        AccessionResolver::FILE_TYPE_SRA,
+        AccessionResolver::FILE_TYPE_FASTQ,
+        AccessionResolver::FILE_TYPE_ALL
+      ].freeze
+      DEFAULT_PROTOCOL = DownloadCandidate::HTTPS_PROTOCOL
+      VALID_PROTOCOLS = [DownloadCandidate::HTTPS_PROTOCOL, DownloadCandidate::FTP_PROTOCOL].freeze
+      # 値が無いことを表す TSV のプレースホルダ。
+      MISSING_VALUE = 'NA'
+
+      def initialize(argv, resolver:, downloader:, stdout:, stderr:, stdin:)
+        @argv = argv
+        @resolver = resolver
+        @downloader = downloader
+        @stdout = stdout
+        @stderr = stderr
+        @stdin = stdin
+        @options = default_options
+        @failed_count = 0
+      end
+
+      def run
+        parse_options
+        return @halt unless @halt.nil?
+
+        begin
+          accessions = collect_accessions
+        rescue MissingAccessionError
+          @stderr.puts build_option_parser
+          return FAILURE_EXIT_STATUS
+        end
+
+        accessions.each do |accession|
+          process(accession)
+        rescue Error => error
+          report_error(error.message, accession: accession)
+          @failed_count += 1
+        end
+        finalize
+        @failed_count.zero? ? SUCCESS_EXIT_STATUS : FAILURE_EXIT_STATUS
+      rescue OptionParser::ParseError, Error => error
+        report_error(error.message)
+        FAILURE_EXIT_STATUS
+      end
+
+      private
+
+      attr_reader :options, :resolver, :downloader, :stdout, :stderr
+
+      # --- サブクラスで上書きするフック -------------------------------------
+
+      # サブコマンド名（バナーやエラー接頭辞に使う）。
+      def command_name
+        raise NotImplementedError
+      end
+
+      # サブクラス固有の既定オプション。
+      def default_options
+        { file_type: DEFAULT_FILE_TYPE, input: nil }
+      end
+
+      # サブクラス固有のオプションを parser に足す。
+      def configure_parser(parser); end
+
+      # サブクラス固有の使用例（1 行ずつ）。
+      def usage_examples
+        []
+      end
+
+      # accession 1 件の処理本体。
+      def process(accession)
+        raise NotImplementedError
+      end
+
+      # 全 accession 処理後の後処理（JSON 出力やサマリなど）。
+      def finalize; end
+
+      # --- 共通処理 ---------------------------------------------------------
+
+      def parse_options
+        parser = build_option_parser
+        parser.parse!(@argv)
+        return unless @halt.nil?
+
+        validate_options
+      end
+
+      def build_option_parser
+        OptionParser.new do |parser|
+          parser.summary_width = 24
+          parser.banner = "Usage: #{Dratools::NAME} #{command_name} [options] [ACCESSION ...]"
+          add_common_options(parser)
+          configure_parser(parser)
+          add_examples(parser)
+        end
+      end
+
+      def add_common_options(parser)
+        file_type_description = "取得対象を指定する。sra, fastq, all (default: #{DEFAULT_FILE_TYPE})"
+        parser.on('--type TYPE', file_type_description) do |value|
+          @options[:file_type] = value
+        end
+        parser.on('-i', '--input FILE', 'accession 一覧をファイルまたは標準入力(-)から読む') do |value|
+          @options[:input] = value
+        end
+        parser.on('-h', '--help', 'ヘルプを表示する') do
+          @stdout.puts parser
+          @halt = SUCCESS_EXIT_STATUS
+        end
+      end
+
+      def add_protocol_option(parser)
+        description = "優先する URL 種別を指定する。https, ftp (default: #{DEFAULT_PROTOCOL})"
+        parser.on('--protocol PROTOCOL', description) { |value| @options[:protocol] = value }
+      end
+
+      def add_examples(parser)
+        examples = usage_examples
+        return if examples.empty?
+
+        parser.separator ''
+        parser.separator 'Examples:'
+        examples.each { |example| parser.separator "  #{example}" }
+      end
+
+      def validate_options
+        file_type = @options[:file_type]
+        return if VALID_FILE_TYPES.include?(file_type)
+
+        raise InvalidOptionError, invalid_message('--type', file_type, VALID_FILE_TYPES)
+      end
+
+      def validate_protocol
+        protocol = @options[:protocol]
+        return if VALID_PROTOCOLS.include?(protocol)
+
+        raise InvalidOptionError, invalid_message('--protocol', protocol, VALID_PROTOCOLS)
+      end
+
+      def invalid_message(name, value, expected)
+        "invalid #{name} '#{value}' (expected: #{expected.join(', ')})"
+      end
+
+      def collect_accessions
+        AccessionInputCollector.new(
+          argv: @argv,
+          input_path: @options[:input],
+          stdin: @stdin
+        ).collect_accessions
+      end
+
+      def resolve_downloads(accession)
+        @resolver.resolve_downloads(accession, file_type: @options[:file_type])
+      end
+
+      # 列名を `#` 始まりのヘッダ行として一度だけ出力する。
+      def emit_tsv_header(columns)
+        return if @tsv_header_printed
+
+        @stdout.puts "##{columns.join("\t")}"
+        @tsv_header_printed = true
+      end
+
+      def report_error(message, accession: nil)
+        prefix = "#{Dratools::NAME} #{command_name}"
+        prefix = "#{prefix}: #{accession}" if accession
+        @stderr.puts "#{prefix}: #{message}"
+      end
+    end
+  end
+end

data/lib/dratools/commands/get_command.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require_relative 'base_command'
+
+module Dratools
+  module Commands
+    # 実際にファイルをダウンロードする。
+    class GetCommand < BaseCommand
+      DEFAULT_OUTPUT_DIRECTORY = DownloadService::DEFAULT_OUTPUT_DIRECTORY
+      DOWNLOAD_SUCCESS_PREFIX = 'Downloaded'
+      DOWNLOAD_SKIPPED_PREFIX = 'Skipped'
+
+      private
+
+      def command_name
+        'get'
+      end
+
+      def default_options
+        super.merge(
+          protocol: DEFAULT_PROTOCOL,
+          outdir: DEFAULT_OUTPUT_DIRECTORY,
+          verify: true,
+          force: false,
+          skip_existing: false
+        )
+      end
+
+      def configure_parser(parser)
+        parser.on('-O', '--outdir DIR',
+                  "ダウンロード先ディレクトリ (default: #{DEFAULT_OUTPUT_DIRECTORY})") do |value|
+          @options[:outdir] = value
+        end
+        add_protocol_option(parser)
+        parser.on('--no-verify', 'md5 がある場合のダウンロード後検証を省略する') { @options[:verify] = false }
+        parser.on('--force', '既存ファイルがあっても再取得する') { @options[:force] = true }
+        parser.on('--skip-existing', '既存ファイルがあれば検証せず再取得しない') { @options[:skip_existing] = true }
+      end
+
+      def usage_examples
+        [
+          "#{Dratools::NAME} get DRR000001",
+          "#{Dratools::NAME} get -O ~/Downloads DRR000001 DRR000002",
+          "#{Dratools::NAME} get --skip-existing -O ~/Downloads DRR000001"
+        ]
+      end
+
+      def validate_options
+        super
+        validate_protocol
+      end
+
+      def process(accession)
+        resolve_downloads(accession).each do |download|
+          save_one_download(accession, download)
+        end
+      end
+
+      def finalize
+        parts = ["#{@downloaded_count.to_i} downloaded", "#{@skipped_count.to_i} skipped"]
+        parts << "#{@failed_count} failed" if @failed_count.positive?
+        @stderr.puts "#{Dratools::NAME} #{command_name}: #{parts.join(', ')}"
+      end
+
+      def save_one_download(accession, download)
+        result = @downloader.save_download(
+          download,
+          outdir: @options[:outdir],
+          protocol: @options[:protocol],
+          verify: @options[:verify],
+          force: @options[:force],
+          skip_existing: @options[:skip_existing]
+        )
+        if result.skipped?
+          @skipped_count = @skipped_count.to_i + 1
+          @stderr.puts "#{DOWNLOAD_SKIPPED_PREFIX}\t#{result.path}"
+        else
+          @downloaded_count = @downloaded_count.to_i + 1
+          @stderr.puts "#{DOWNLOAD_SUCCESS_PREFIX}\t#{result.path}"
+        end
+      rescue Error => error
+        report_error(error.message, accession: accession)
+        @failed_count += 1
+      end
+    end
+  end
+end