dratools 0.0.1 → 0.0.3

data/lib/dratools/ddbj_resource_client.rb

@@ -7,12 +7,18 @@ require 'uri'
 
 require_relative 'errors'
 require_relative 'version'
+require_relative 'ddbj_record_fields'
+require_relative 'progress_reporter'
 
 module Dratools
-  # DDBJ resource API を呼び出す薄い HTTP クライアント。
+  # DDBJ Search API を呼び出す薄い HTTP クライアント。
   class DdbjResourceClient
-    DDBJ_RESOURCE_BASE_URL = 'https://ddbj.nig.ac.jp/resource'
-    RESOURCE_RECORD_EXTENSION = '.json'
+    DDBJ_SEARCH_API_BASE_URL = 'https://ddbj.nig.ac.jp/search/api'
+    ENTRIES_PATH = 'entries'
+    DBLINK_PATH = 'dblink'
+    ENTRY_RECORD_EXTENSION = '.json'
+    BULK_MAX_IDS = 1000
+    DBLINK_COUNTS_MAX_ITEMS = 100
     HTTPS_SCHEME = 'https'
     HTTP_LOCATION_HEADER = 'location'
     USER_AGENT_HEADER = 'User-Agent'
@@ -20,40 +26,106 @@ module Dratools
     DEFAULT_OPEN_TIMEOUT_SECONDS = 10
     DEFAULT_READ_TIMEOUT_SECONDS = 30
 
-    def initialize(base_url: DDBJ_RESOURCE_BASE_URL, open_timeout: DEFAULT_OPEN_TIMEOUT_SECONDS,
-                   read_timeout: DEFAULT_READ_TIMEOUT_SECONDS)
+    def initialize(base_url: DDBJ_SEARCH_API_BASE_URL, open_timeout: DEFAULT_OPEN_TIMEOUT_SECONDS,
+                   read_timeout: DEFAULT_READ_TIMEOUT_SECONDS, progress: ProgressReporter.new)
       @base_url = base_url.delete_suffix('/')
       @open_timeout = open_timeout
       @read_timeout = read_timeout
+      @progress = progress
     end
 
     def fetch_resource_record(type, accession)
-      fetch_json("#{@base_url}/#{type}/#{accession}#{RESOURCE_RECORD_EXTENSION}")
+      @progress.report("fetching #{type} #{accession}")
+      fetch_json("#{@base_url}/#{ENTRIES_PATH}/#{type}/#{accession}#{ENTRY_RECORD_EXTENSION}")
+    end
+
+    def fetch_db_links(type, accession, target: nil)
+      @progress.report("linking #{type} #{accession}")
+      request_uri = URI("#{@base_url}/#{DBLINK_PATH}/#{type}/#{accession}")
+      request_uri.query = URI.encode_www_form(target: target) if target
+      fetch_json(request_uri.to_s).fetch(DdbjRecordFields::DB_XREFS_KEY, [])
+    end
+
+    def fetch_resource_records_bulk(type, accessions, include_db_xrefs: false)
+      accessions.each_slice(BULK_MAX_IDS).with_object({}) do |chunk, records|
+        records.merge!(
+          fetch_resource_records_bulk_chunk(type, chunk, include_db_xrefs: include_db_xrefs)
+        )
+      end
+    end
+
+    def fetch_db_link_counts(items)
+      items.each_slice(DBLINK_COUNTS_MAX_ITEMS).with_object({}) do |chunk, counts|
+        counts.merge!(fetch_db_link_counts_chunk(chunk))
+      end
     end
 
     private
 
+    def fetch_db_link_counts_chunk(items)
+      @progress.report("counting links (#{items.length})")
+      request_url = "#{@base_url}/#{DBLINK_PATH}/counts"
+      payload = post_json(request_url, items: items)
+      payload.fetch('items', []).to_h do |item|
+        [[item['type'], item[DdbjRecordFields::IDENTIFIER_KEY]], item.fetch('counts', {})]
+      end
+    end
+
+    def fetch_resource_records_bulk_chunk(type, accessions, include_db_xrefs:)
+      @progress.report("fetching #{type} bulk (#{accessions.length})")
+      request_uri = URI("#{@base_url}/#{ENTRIES_PATH}/#{type}/bulk")
+      request_uri.query = URI.encode_www_form(includeDbXrefs: include_db_xrefs)
+      payload = post_json(request_uri.to_s, ids: accessions)
+      payload.fetch('entries', []).to_h do |record|
+        accession = record[DdbjRecordFields::IDENTIFIER_KEY] ||
+                    record[DdbjRecordFields::ACCESSION_KEY] ||
+                    record[DdbjRecordFields::ID_KEY] ||
+                    record[DdbjRecordFields::PRIMARY_ID_KEY]
+        [accession, record]
+      end
+    end
+
     def fetch_json(request_url, redirects_remaining = DEFAULT_REDIRECT_LIMIT)
-      request_uri = URI(request_url)
-      response = get_http_response(request_uri)
-
-      case response
-      when Net::HTTPSuccess
-        JSON.parse(response.body)
-      when Net::HTTPRedirection
-        raise NetworkError, "too many redirects: #{request_url}" if redirects_remaining <= 0
-
-        location = response[HTTP_LOCATION_HEADER]
-        raise NetworkError, "redirect without location: #{request_url}" if location.to_s.empty?
-
-        fetch_json(URI.join(request_uri, location).to_s, redirects_remaining - 1)
-      when Net::HTTPNotFound
-        raise NotFoundError, "not found: #{request_url}"
-      else
+      with_network_errors(request_url) do
+        request_uri = URI(request_url)
+        response = get_http_response(request_uri)
+
+        case response
+        when Net::HTTPSuccess
+          parse_json_response(response, request_url)
+        when Net::HTTPRedirection
+          raise NetworkError, "too many redirects: #{request_url}" if redirects_remaining <= 0
+
+          location = response[HTTP_LOCATION_HEADER]
+          raise NetworkError, "redirect without location: #{request_url}" if location.to_s.empty?
+
+          fetch_json(URI.join(request_uri, location).to_s, redirects_remaining - 1)
+        when Net::HTTPNotFound
+          raise NotFoundError, "not found: #{request_url}"
+        else
+          raise NetworkError, "HTTP #{response.code}: #{request_url}"
+        end
+      end
+    end
+
+    def post_json(request_url, payload)
+      with_network_errors(request_url) do
+        response = post_http_response(URI(request_url), payload)
+        return parse_json_response(response, request_url) if response.is_a?(Net::HTTPSuccess)
+        raise NotFoundError, "not found: #{request_url}" if response.is_a?(Net::HTTPNotFound)
+
         raise NetworkError, "HTTP #{response.code}: #{request_url}"
       end
+    end
+
+    def parse_json_response(response, request_url)
+      JSON.parse(response.body)
     rescue JSON::ParserError => error
       raise NetworkError, "invalid JSON from #{request_url}: #{error.message}", cause: error
+    end
+
+    def with_network_errors(request_url)
+      yield
     rescue Timeout::Error, IOError, SocketError, SystemCallError => error
       message = "failed to fetch #{request_url}: #{error.class}: #{error.message}"
       raise NetworkError, message, cause: error
@@ -71,6 +143,21 @@ module Dratools
       end
     end
 
+    def post_http_response(request_uri, payload)
+      Net::HTTP.start(
+        request_uri.host,
+        request_uri.port,
+        use_ssl: request_uri.scheme == HTTPS_SCHEME,
+        open_timeout: @open_timeout,
+        read_timeout: @read_timeout
+      ) do |http|
+        request = Net::HTTP::Post.new(request_uri.request_uri, USER_AGENT_HEADER => user_agent)
+        request['Content-Type'] = 'application/json'
+        request.body = JSON.generate(payload)
+        http.request(request)
+      end
+    end
+
     def user_agent
       "#{NAME}/#{VERSION}"
     end

data/lib/dratools/download_candidate_builder.rb

@@ -4,12 +4,12 @@ require_relative 'ddbj_record_fields'
 require_relative 'download_candidate'
 
 module Dratools
-  # DDBJ run レコードの downloadUrl/distribution から DownloadCandidate を作る。
+  # DDBJ run レコードの distribution から DownloadCandidate を作る。
   class DownloadCandidateBuilder
     def build_from_run_record(run_record)
       run_accession = run_accession_from(run_record)
       downloads = download_items_from(run_record).filter_map do |download_item|
-        build_from_download_item(run_accession, download_item)
+        build_from_distribution_item(run_accession, download_item)
       end
       downloads.uniq { |download| download_key(download) }
     end
@@ -24,21 +24,12 @@ module Dratools
     end
 
     def download_items_from(ddbj_record)
-      ddbj_record.fetch(DdbjRecordFields::DOWNLOAD_URL_KEY, []) +
-        ddbj_record.fetch(DdbjRecordFields::DISTRIBUTION_KEY, [])
+      Array(ddbj_record[DdbjRecordFields::DISTRIBUTION_KEY])
     end
 
-    def build_from_download_item(run_accession, download_item)
+    def build_from_distribution_item(run_accession, download_item)
       return unless download_item.is_a?(Hash)
 
-      if download_item[DdbjRecordFields::CONTENT_URL_KEY]
-        build_from_distribution_item(run_accession, download_item)
-      else
-        build_from_download_url_item(run_accession, download_item)
-      end
-    end
-
-    def build_from_distribution_item(run_accession, download_item)
       file_type = file_type_from_distribution(download_item)
       return unless file_type
 
@@ -52,28 +43,10 @@ module Dratools
       )
     end
 
-    def build_from_download_url_item(run_accession, download_item)
-      file_type = file_type_from_download_url(download_item)
-      return unless file_type
-
-      DownloadCandidate.new(
-        run_accession: run_accession,
-        type: file_type,
-        url: download_item[DdbjRecordFields::URL_KEY],
-        ftp_url: download_item[DdbjRecordFields::FTP_URL_KEY],
-        size: download_item[DdbjRecordFields::SIZE_KEY] || download_item[DdbjRecordFields::FILE_SIZE_KEY],
-        md5: download_item[DdbjRecordFields::MD5_KEY] || download_item[DdbjRecordFields::MD5_SUM_KEY]
-      )
-    end
-
     def file_type_from_distribution(download_item)
       file_type_from(download_item[DdbjRecordFields::ENCODING_FORMAT_KEY])
     end
 
-    def file_type_from_download_url(download_item)
-      file_type_from(download_item[DdbjRecordFields::TYPE_KEY])
-    end
-
     def file_type_from(value)
       case value.to_s.downcase
       when DdbjRecordFields::FILE_TYPE_SRA

data/lib/dratools/external_command_runner.rb

@@ -8,77 +8,146 @@ require_relative 'config'
 require_relative 'errors'
 
 module Dratools
-  # curl か wget を使って URL の確認とダウンロードを行うラッパー。
+  # curl, wget, aria2c のいずれかを使って URL の確認とダウンロードを行うラッパー。
   #
   # probe は短時間・無出力で済ませ、download は外部コマンドの進捗を端末へ流す。
   # 巨大ファイルを扱うため、download には総時間制限ではなく失速検知を使う。
   class ExternalCommandRunner
     CURL_COMMAND = 'curl'
     WGET_COMMAND = 'wget'
-    SUPPORTED_COMMANDS = [CURL_COMMAND, WGET_COMMAND].freeze
+    ARIA2_COMMAND = 'aria2c'
+    AUTO_COMMANDS = [CURL_COMMAND, WGET_COMMAND].freeze
+    SUPPORTED_COMMANDS = [*AUTO_COMMANDS, ARIA2_COMMAND].freeze
+
     COMMAND_NOT_FOUND_MESSAGE = 'curl または wget が見つかりません'
+    PREFERRED_COMMAND_NOT_FOUND_MESSAGE = '指定されたダウンロードコマンドが見つかりません'
+    UNSUPPORTED_COMMAND_MESSAGE = '未対応のダウンロードコマンドです'
     DEFAULT_PROBE_TIMEOUT_SECONDS = 5
     PROBE_BYTE_RANGE = '0-0'
     SINGLE_ATTEMPT_COUNT = 1
 
+    # リダイレクト、HTTP エラー、静かな probe、範囲取得。
     CURL_PROBE_OPTIONS = ['--location', '--fail', '--silent', '--show-error', '--range'].freeze
+    # probe 全体のタイムアウト。
     CURL_TIMEOUT_OPTION = '--max-time'
     CURL_CONNECT_TIMEOUT_OPTION = '--connect-timeout'
+    # 失速判定に使う最低転送速度。
     CURL_SPEED_LIMIT_OPTION = '--speed-limit'
+    # 最低速度を下回ってよい秒数。
     CURL_SPEED_TIME_OPTION = '--speed-time'
     CURL_RETRY_OPTION = '--retry'
     CURL_OUTPUT_OPTION = '--output'
+    # 部分ファイルがあれば続きから再開。
     CURL_DOWNLOAD_OPTIONS = ['--location', '--fail', '--continue-at', '-'].freeze
 
+    # probe でファイルを保存しない。
     WGET_PROBE_OPTIONS = ['--spider'].freeze
+    # probe 全体のタイムアウト。
     WGET_TIMEOUT_OPTION = '--timeout'
     WGET_CONNECT_TIMEOUT_OPTION = '--connect-timeout'
+    # wget で失速検知に近い意味で使う。
     WGET_READ_TIMEOUT_OPTION = '--read-timeout'
+    # probe では 1 回だけにする。
     WGET_TRIES_OPTION = '--tries'
     WGET_WAITRETRY_OPTION = '--waitretry'
+    # 部分ファイルがあれば続きから再開。
     WGET_CONTINUE_OPTION = '--continue'
     WGET_OUTPUT_OPTION = '--output-document'
 
-    def initialize(preferred: nil)
+    # probe で保存せず、通常出力も抑える。
+    ARIA2_PROBE_OPTIONS = ['--dry-run=true', '--quiet=true'].freeze
+    ARIA2_CONNECT_TIMEOUT_OPTION = '--connect-timeout'
+    # aria2c で失速検知に近い意味で使う。
+    ARIA2_TIMEOUT_OPTION = '--timeout'
+    # curl の --speed-limit に相当する最低転送速度。
+    ARIA2_LOWEST_SPEED_LIMIT_OPTION = '--lowest-speed-limit'
+    ARIA2_MAX_TRIES_OPTION = '--max-tries'
+    ARIA2_RETRY_WAIT_OPTION = '--retry-wait'
+    # 部分ファイルがあれば続きから再開。
+    ARIA2_CONTINUE_OPTION = '--continue=true'
+    # aria2c は保存先をディレクトリとファイル名に分ける。
+    ARIA2_DIR_OPTION = '--dir'
+    ARIA2_OUT_OPTION = '--out'
+    # 既定では並列取得しない。
+    ARIA2_SINGLE_CONNECTION_OPTIONS = ['--split=1', '--max-connection-per-server=1'].freeze
+
+    def initialize(preferred: Config.download_command)
       @preferred = preferred
     end
 
     def available_command
-      candidates = [@preferred, *SUPPORTED_COMMANDS].compact
+      candidates = @preferred ? [@preferred] : AUTO_COMMANDS
       candidates.find { |command_name| executable_command?(command_name) }
     end
 
     def probe_url(url, timeout: DEFAULT_PROBE_TIMEOUT_SECONDS)
-      tool = available_command || raise(CommandError, COMMAND_NOT_FOUND_MESSAGE)
+      tool = available_command || raise(CommandError, command_not_found_message)
       # 巨大ファイルを落とさないよう、短時間・最小範囲の確認に留める。
       command =
-        if File.basename(tool) == CURL_COMMAND
+        case File.basename(tool)
+        when CURL_COMMAND
+          # 例: curl --location --fail --silent --show-error --range 0-0
+          #          --max-time 5 --output /dev/null URL
           [tool, *CURL_PROBE_OPTIONS, PROBE_BYTE_RANGE, CURL_TIMEOUT_OPTION, timeout.to_s,
            CURL_OUTPUT_OPTION, null_device, url]
-        else
+        when WGET_COMMAND
+          # 例: wget --spider --timeout=5 --tries=1 URL
           [tool, *WGET_PROBE_OPTIONS, "#{WGET_TIMEOUT_OPTION}=#{timeout}",
            "#{WGET_TRIES_OPTION}=#{SINGLE_ATTEMPT_COUNT}", url]
+        when ARIA2_COMMAND
+          # 例: aria2c --dry-run=true --quiet=true --connect-timeout=5 --timeout=5 --max-tries=1 URL
+          [tool, *ARIA2_PROBE_OPTIONS,
+           "#{ARIA2_CONNECT_TIMEOUT_OPTION}=#{timeout}",
+           "#{ARIA2_TIMEOUT_OPTION}=#{timeout}",
+           "#{ARIA2_MAX_TRIES_OPTION}=#{SINGLE_ATTEMPT_COUNT}", url]
+        else
+          unsupported_command!(tool)
         end
       run_quietly(command)
     end
 
     def download_url(url, output_path)
-      tool = available_command || raise(CommandError, COMMAND_NOT_FOUND_MESSAGE)
+      tool = available_command || raise(CommandError, command_not_found_message)
       command =
-        if File.basename(tool) == CURL_COMMAND
+        case File.basename(tool)
+        when CURL_COMMAND
+          # curl の低速検知は「指定秒数のあいだ指定速度を下回ったら失敗」。
+          # ネットワークが完全に切れず低速で固まるケースを、総時間制限なしで検出する。
+          # 例: curl --location --fail --continue-at - --connect-timeout 30
+          #          --speed-limit 1024 --speed-time 60 --retry 3 --output OUT URL
           [tool, *CURL_DOWNLOAD_OPTIONS,
            CURL_CONNECT_TIMEOUT_OPTION, Config.download_connect_timeout_seconds.to_s,
            CURL_SPEED_LIMIT_OPTION, Config.download_stall_speed_bytes_per_second.to_s,
            CURL_SPEED_TIME_OPTION, Config.download_stall_timeout_seconds.to_s,
            CURL_RETRY_OPTION, Config.download_retry_count.to_s,
            CURL_OUTPUT_OPTION, output_path, url]
-        else
+        when WGET_COMMAND
+          # wget では --read-timeout を失速検知に近い意味で使う。
+          # --continue は部分ファイルの続きから再開し、--output-document は保存先を固定する。
+          # 例: wget --continue --connect-timeout=30 --read-timeout=60
+          #          --tries=4 --waitretry=5 --output-document OUT URL
           [tool, WGET_CONTINUE_OPTION,
            "#{WGET_CONNECT_TIMEOUT_OPTION}=#{Config.download_connect_timeout_seconds}",
            "#{WGET_READ_TIMEOUT_OPTION}=#{Config.download_stall_timeout_seconds}",
-           "#{WGET_TRIES_OPTION}=#{Config.download_retry_count}",
+           "#{WGET_TRIES_OPTION}=#{download_attempt_count}",
            "#{WGET_WAITRETRY_OPTION}=#{Config.download_retry_wait_seconds}",
            WGET_OUTPUT_OPTION, output_path, url]
+        when ARIA2_COMMAND
+          # aria2c は保存先をディレクトリとファイル名に分けて指定する。
+          # --continue=true は部分ファイルがあれば続きから再開する。
+          # 例: aria2c --continue=true --split=1 --max-connection-per-server=1
+          #          --connect-timeout=30 --timeout=60 --lowest-speed-limit=1024
+          #          --max-tries=4 --retry-wait=5 --dir DIR --out FILE URL
+          [tool, ARIA2_CONTINUE_OPTION, *ARIA2_SINGLE_CONNECTION_OPTIONS,
+           "#{ARIA2_CONNECT_TIMEOUT_OPTION}=#{Config.download_connect_timeout_seconds}",
+           "#{ARIA2_TIMEOUT_OPTION}=#{Config.download_stall_timeout_seconds}",
+           "#{ARIA2_LOWEST_SPEED_LIMIT_OPTION}=#{Config.download_stall_speed_bytes_per_second}",
+           "#{ARIA2_MAX_TRIES_OPTION}=#{download_attempt_count}",
+           "#{ARIA2_RETRY_WAIT_OPTION}=#{Config.download_retry_wait_seconds}",
+           "#{ARIA2_DIR_OPTION}=#{File.dirname(output_path)}",
+           "#{ARIA2_OUT_OPTION}=#{File.basename(output_path)}", url]
+        else
+          unsupported_command!(tool)
         end
       run_streaming(command)
     end
@@ -93,7 +162,7 @@ module Dratools
     end
 
     def run_streaming(command)
-      # 配列形式で渡すことでシェルを介さず、curl/wget の stderr 進捗はそのまま見せる。
+      # 配列形式で渡すことでシェルを介さず、外部コマンドの stderr 進捗はそのまま見せる。
       return true if system(*command)
 
       status = $CHILD_STATUS
@@ -108,6 +177,20 @@ module Dratools
       end
     end
 
+    def command_not_found_message
+      return "#{PREFERRED_COMMAND_NOT_FOUND_MESSAGE}: #{@preferred}" if @preferred
+
+      COMMAND_NOT_FOUND_MESSAGE
+    end
+
+    def download_attempt_count
+      Config.download_retry_count + 1
+    end
+
+    def unsupported_command!(tool)
+      raise CommandError, "#{UNSUPPORTED_COMMAND_MESSAGE}: #{tool}"
+    end
+
     def null_device
       File::NULL
     end

data/lib/dratools/progress_reporter.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+module Dratools
+  # 対話的な端末のときだけ stderr へ一行進捗を表示する軽量レポーター。
+  # データは stdout に出るため、stdout をパイプしても進捗は混ざらない。
+  # 非 TTY（リダイレクト・パイプ・CI）では制御文字を出さず完全に無音にする。
+  class ProgressReporter
+    CLEAR_LINE = "\r\e[K"
+
+    # 通常の出力の直前に、表示中の進捗行を消す IO ラッパー。
+    class ClearingIO
+      def initialize(io, reporter)
+        @io = io
+        @reporter = reporter
+      end
+
+      def puts(*args)
+        @reporter.finish
+        @io.puts(*args)
+      end
+
+      def print(*args)
+        @reporter.finish
+        @io.print(*args)
+      end
+
+      def write(*args)
+        @reporter.finish
+        @io.write(*args)
+      end
+
+      def flush
+        @io.flush
+      end
+
+      def tty?
+        @io.respond_to?(:tty?) && @io.tty?
+      end
+
+      def method_missing(name, ...)
+        return super unless @io.respond_to?(name)
+
+        @io.public_send(name, ...)
+      end
+
+      def respond_to_missing?(name, include_private = false)
+        @io.respond_to?(name, include_private) || super
+      end
+    end
+
+    def initialize(io: $stderr, enabled: nil)
+      @io = io
+      @enabled = enabled.nil? ? interactive?(io) : enabled
+      @count = 0
+      @active = false
+    end
+
+    def clearing_io(io = @io)
+      ClearingIO.new(io, self)
+    end
+
+    # 1 件の進捗を表示する。直前の行を消してから上書きする。
+    def report(label)
+      return unless @enabled
+
+      @count += 1
+      @io.print("#{CLEAR_LINE}#{label} (#{@count})")
+      @io.flush
+      @active = true
+    end
+
+    # 残っている進捗行を消す。コマンド終了時（成功・失敗どちらでも）に呼ぶ。
+    def finish
+      return unless @active
+
+      @io.print(CLEAR_LINE)
+      @io.flush
+      @active = false
+    end
+
+    private
+
+    def interactive?(io)
+      io.respond_to?(:tty?) && io.tty?
+    end
+  end
+end

data/lib/dratools/run_record_collector.rb

@@ -10,7 +10,6 @@ require_relative 'traversal_node'
 module Dratools
   # BioProject などの上位レコードから DDBJ sra-run レコードを集める。
   class RunRecordCollector
-    XREF_URL_PATTERN = %r{/(?:resource|search/entry)/([^/]+)/([^/?#.]+)}
     TRAVERSABLE_XREF_TYPES = [
       DdbjRecordFields::SRA_RUN_RESOURCE_TYPE,
       DdbjRecordFields::SRA_EXPERIMENT_RESOURCE_TYPE,
@@ -40,14 +39,26 @@ module Dratools
         return node
       end
 
-      direct_children = explore_edges(run_xrefs, TraversalNode::DB_XREF_RELATION, seen_keys,
-                                      tolerant: tolerant,
-                                      direct_run_fetch_limit: direct_run_fetch_limit)
+      direct_children = explore_run_xrefs(
+        run_xrefs,
+        seen_keys,
+        tolerant: tolerant,
+        direct_run_fetch_limit: direct_run_fetch_limit
+      )
       if direct_children.any? { |child| child.run? || child.run_records.any? }
         node.children.concat(direct_children)
         return node
       end
 
+      node.children.concat(
+        recursive_children(ddbj_record, xrefs, seen_keys, tolerant, direct_run_fetch_limit)
+      )
+      node
+    end
+
+    private
+
+    def recursive_children(ddbj_record, xrefs, seen_keys, tolerant, direct_run_fetch_limit)
       recursive_xrefs = xrefs.select { |xref| traversable_xref?(xref) }
       validate_recursive_non_run_xref_count!(ddbj_record, recursive_xrefs)
       db_xref_edges = explore_edges(
@@ -64,12 +75,9 @@ module Dratools
         tolerant: tolerant,
         direct_run_fetch_limit: direct_run_fetch_limit
       )
-      node.children.concat(db_xref_edges + child_edges)
-      node
+      db_xref_edges + child_edges
     end
 
-    private
-
     def lightweight_direct_run_nodes(run_xrefs, direct_run_fetch_limit)
       return nil unless direct_run_fetch_limit && run_xrefs.length > direct_run_fetch_limit
 
@@ -86,7 +94,8 @@ module Dratools
       accession = record_accession(ddbj_record) || 'record'
       raise InvalidRecordError,
             "#{accession} has #{non_run_xrefs.length} linked non-run records; " \
-            'refine to an experiment/sample accession before run expansion'
+            'refine to an experiment/sample accession before run expansion, ' \
+            "or set #{Config::MAX_RECURSIVE_NON_RUN_XREFS_ENV}=unlimited"
     end
 
     def child_bioprojects(ddbj_record)
@@ -111,6 +120,53 @@ module Dratools
       end
     end
 
+    def explore_run_xrefs(run_xrefs, seen_keys, tolerant:, direct_run_fetch_limit:)
+      fetchable_xrefs = unseen_fetchable_xrefs(run_xrefs, seen_keys)
+      return [] if fetchable_xrefs.empty?
+
+      accessions = fetchable_xrefs.map { |xref| xref_accession(xref) }
+      records = @client.fetch_resource_records_bulk(
+        DdbjRecordFields::SRA_RUN_RESOURCE_TYPE,
+        accessions,
+        include_db_xrefs: false
+      )
+      fetchable_xrefs.map do |xref|
+        accession = xref_accession(xref)
+        if (record = records[accession])
+          explore(
+            record,
+            seen_keys: seen_keys,
+            relation: TraversalNode::DB_XREF_RELATION,
+            tolerant: tolerant,
+            direct_run_fetch_limit: direct_run_fetch_limit
+          )
+        elsif tolerant
+          node_from_xref(
+            xref,
+            relation: TraversalNode::DB_XREF_RELATION,
+            error: "not found: #{accession}"
+          )
+        else
+          raise NotFoundError, "not found: sra-run/#{accession}"
+        end
+      end
+    end
+
+    def unseen_fetchable_xrefs(xrefs, seen_keys)
+      xrefs.each_with_object([]) do |xref, selected|
+        next unless traversable_xref?(xref)
+
+        accession = xref_accession(xref)
+        next if accession.empty?
+
+        reference_key = xref_key(xref)
+        next if reference_key.empty? || seen_keys.include?(reference_key)
+
+        seen_keys.add(reference_key)
+        selected << xref
+      end
+    end
+
     def explore_xref(xref, relation, seen_keys, tolerant:, direct_run_fetch_limit:)
       linked_record = fetch_xref_record(xref)
       explore(
@@ -127,20 +183,17 @@ module Dratools
     end
 
     def xref_key(xref)
-      reference_url = xref[DdbjRecordFields::URL_KEY].to_s
-      return reference_url unless reference_url.empty?
+      xref_accession(xref)
+    end
 
+    def xref_accession(xref)
       (xref[DdbjRecordFields::ID_KEY] || xref[DdbjRecordFields::IDENTIFIER_KEY]).to_s
     end
 
     def fetch_xref_record(xref)
-      if (resource_match = xref[DdbjRecordFields::URL_KEY].to_s.match(XREF_URL_PATTERN))
-        @client.fetch_resource_record(resource_match[1], resource_match[2])
-      elsif xref[DdbjRecordFields::ID_KEY] || xref[DdbjRecordFields::IDENTIFIER_KEY]
-        fetch_xref_by_identifier(xref)
-      else
-        raise InvalidRecordError, 'sra-run xref has no URL or id'
-      end
+      raise InvalidRecordError, 'xref has no identifier' if xref_accession(xref).empty?
+
+      fetch_xref_by_identifier(xref)
     end
 
     def fetch_xref_by_identifier(xref)
@@ -160,15 +213,13 @@ module Dratools
 
     def run_record?(ddbj_record)
       record_type = ddbj_record[DdbjRecordFields::TYPE_KEY]
-      return record_type == DdbjRecordFields::SRA_RUN_RESOURCE_TYPE if record_type
-
-      ddbj_record[DdbjRecordFields::DOWNLOAD_URL_KEY].is_a?(Array)
+      record_type == DdbjRecordFields::SRA_RUN_RESOURCE_TYPE
     end
 
     def node_from_record(ddbj_record, relation:)
       TraversalNode.new(
         relation: relation,
-        type: ddbj_record[DdbjRecordFields::TYPE_KEY] || inferred_record_type(ddbj_record),
+        type: ddbj_record[DdbjRecordFields::TYPE_KEY],
         accession: record_accession(ddbj_record),
         object_type: ddbj_record['objectType'],
         record: run_record?(ddbj_record) ? ddbj_record : nil
@@ -190,9 +241,5 @@ module Dratools
         ddbj_record[DdbjRecordFields::ID_KEY] ||
         ddbj_record[DdbjRecordFields::PRIMARY_ID_KEY]
     end
-
-    def inferred_record_type(ddbj_record)
-      DdbjRecordFields::SRA_RUN_RESOURCE_TYPE if run_record?(ddbj_record)
-    end
   end
 end