dratools 0.0.1

data/lib/dratools/run_record_collector.rb

@@ -0,0 +1,198 @@
+# frozen_string_literal: true
+
+require 'set'
+
+require_relative 'config'
+require_relative 'ddbj_record_fields'
+require_relative 'errors'
+require_relative 'traversal_node'
+
+module Dratools
+  # BioProject などの上位レコードから DDBJ sra-run レコードを集める。
+  class RunRecordCollector
+    XREF_URL_PATTERN = %r{/(?:resource|search/entry)/([^/]+)/([^/?#.]+)}
+    TRAVERSABLE_XREF_TYPES = [
+      DdbjRecordFields::SRA_RUN_RESOURCE_TYPE,
+      DdbjRecordFields::SRA_EXPERIMENT_RESOURCE_TYPE,
+      DdbjRecordFields::SRA_SAMPLE_RESOURCE_TYPE,
+      DdbjRecordFields::SRA_STUDY_RESOURCE_TYPE,
+      DdbjRecordFields::SRA_SUBMISSION_RESOURCE_TYPE,
+      DdbjRecordFields::BIOPROJECT_RESOURCE_TYPE,
+      DdbjRecordFields::BIOSAMPLE_RESOURCE_TYPE
+    ].freeze
+    def initialize(client:)
+      @client = client
+    end
+
+    def collect_run_records(ddbj_record, seen_keys = Set.new)
+      explore(ddbj_record, seen_keys: seen_keys).run_records
+    end
+
+    def explore(ddbj_record, seen_keys: Set.new, relation: TraversalNode::ROOT_RELATION,
+                tolerant: false, direct_run_fetch_limit: nil)
+      node = node_from_record(ddbj_record, relation: relation)
+      return node if run_record?(ddbj_record)
+
+      xrefs = ddbj_record.fetch(DdbjRecordFields::DB_XREFS_KEY, [])
+      run_xrefs = xrefs.select { |xref| sra_run_xref?(xref) }
+      if (lightweight_children = lightweight_direct_run_nodes(run_xrefs, direct_run_fetch_limit))
+        node.children.concat(lightweight_children)
+        return node
+      end
+
+      direct_children = explore_edges(run_xrefs, TraversalNode::DB_XREF_RELATION, seen_keys,
+                                      tolerant: tolerant,
+                                      direct_run_fetch_limit: direct_run_fetch_limit)
+      if direct_children.any? { |child| child.run? || child.run_records.any? }
+        node.children.concat(direct_children)
+        return node
+      end
+
+      recursive_xrefs = xrefs.select { |xref| traversable_xref?(xref) }
+      validate_recursive_non_run_xref_count!(ddbj_record, recursive_xrefs)
+      db_xref_edges = explore_edges(
+        recursive_xrefs,
+        TraversalNode::DB_XREF_RELATION,
+        seen_keys,
+        tolerant: tolerant,
+        direct_run_fetch_limit: direct_run_fetch_limit
+      )
+      child_edges = explore_edges(
+        child_bioprojects(ddbj_record),
+        TraversalNode::CHILD_BIOPROJECT_RELATION,
+        seen_keys,
+        tolerant: tolerant,
+        direct_run_fetch_limit: direct_run_fetch_limit
+      )
+      node.children.concat(db_xref_edges + child_edges)
+      node
+    end
+
+    private
+
+    def lightweight_direct_run_nodes(run_xrefs, direct_run_fetch_limit)
+      return nil unless direct_run_fetch_limit && run_xrefs.length > direct_run_fetch_limit
+
+      run_xrefs.map { |xref| node_from_xref(xref, relation: TraversalNode::DB_XREF_RELATION) }
+    end
+
+    def validate_recursive_non_run_xref_count!(ddbj_record, xrefs)
+      max_xrefs = Config.max_recursive_non_run_xrefs
+      return unless max_xrefs
+
+      non_run_xrefs = xrefs.reject { |xref| sra_run_xref?(xref) }
+      return if non_run_xrefs.length <= max_xrefs
+
+      accession = record_accession(ddbj_record) || 'record'
+      raise InvalidRecordError,
+            "#{accession} has #{non_run_xrefs.length} linked non-run records; " \
+            'refine to an experiment/sample accession before run expansion'
+    end
+
+    def child_bioprojects(ddbj_record)
+      ddbj_record.fetch(DdbjRecordFields::CHILD_BIOPROJECTS_KEY, [])
+    end
+
+    def explore_edges(xrefs, relation, seen_keys, tolerant:, direct_run_fetch_limit:)
+      xrefs.each_with_object([]) do |xref, children|
+        next unless traversable_xref?(xref)
+
+        reference_key = xref_key(xref)
+        next if reference_key.empty? || seen_keys.include?(reference_key)
+
+        seen_keys.add(reference_key)
+        children << explore_xref(
+          xref,
+          relation,
+          seen_keys,
+          tolerant: tolerant,
+          direct_run_fetch_limit: direct_run_fetch_limit
+        )
+      end
+    end
+
+    def explore_xref(xref, relation, seen_keys, tolerant:, direct_run_fetch_limit:)
+      linked_record = fetch_xref_record(xref)
+      explore(
+        linked_record,
+        seen_keys: seen_keys,
+        relation: relation,
+        tolerant: tolerant,
+        direct_run_fetch_limit: direct_run_fetch_limit
+      )
+    rescue Error => error
+      raise unless tolerant
+
+      node_from_xref(xref, relation: relation, error: error.message)
+    end
+
+    def xref_key(xref)
+      reference_url = xref[DdbjRecordFields::URL_KEY].to_s
+      return reference_url unless reference_url.empty?
+
+      (xref[DdbjRecordFields::ID_KEY] || xref[DdbjRecordFields::IDENTIFIER_KEY]).to_s
+    end
+
+    def fetch_xref_record(xref)
+      if (resource_match = xref[DdbjRecordFields::URL_KEY].to_s.match(XREF_URL_PATTERN))
+        @client.fetch_resource_record(resource_match[1], resource_match[2])
+      elsif xref[DdbjRecordFields::ID_KEY] || xref[DdbjRecordFields::IDENTIFIER_KEY]
+        fetch_xref_by_identifier(xref)
+      else
+        raise InvalidRecordError, 'sra-run xref has no URL or id'
+      end
+    end
+
+    def fetch_xref_by_identifier(xref)
+      @client.fetch_resource_record(
+        xref[DdbjRecordFields::TYPE_KEY],
+        xref[DdbjRecordFields::ID_KEY] || xref[DdbjRecordFields::IDENTIFIER_KEY]
+      )
+    end
+
+    def sra_run_xref?(xref)
+      xref[DdbjRecordFields::TYPE_KEY] == DdbjRecordFields::SRA_RUN_RESOURCE_TYPE
+    end
+
+    def traversable_xref?(xref)
+      TRAVERSABLE_XREF_TYPES.include?(xref[DdbjRecordFields::TYPE_KEY])
+    end
+
+    def run_record?(ddbj_record)
+      record_type = ddbj_record[DdbjRecordFields::TYPE_KEY]
+      return record_type == DdbjRecordFields::SRA_RUN_RESOURCE_TYPE if record_type
+
+      ddbj_record[DdbjRecordFields::DOWNLOAD_URL_KEY].is_a?(Array)
+    end
+
+    def node_from_record(ddbj_record, relation:)
+      TraversalNode.new(
+        relation: relation,
+        type: ddbj_record[DdbjRecordFields::TYPE_KEY] || inferred_record_type(ddbj_record),
+        accession: record_accession(ddbj_record),
+        object_type: ddbj_record['objectType'],
+        record: run_record?(ddbj_record) ? ddbj_record : nil
+      )
+    end
+
+    def node_from_xref(xref, relation:, error: nil)
+      TraversalNode.new(
+        relation: relation,
+        type: xref[DdbjRecordFields::TYPE_KEY],
+        accession: xref[DdbjRecordFields::ID_KEY] || xref[DdbjRecordFields::IDENTIFIER_KEY],
+        error: error
+      )
+    end
+
+    def record_accession(ddbj_record)
+      ddbj_record[DdbjRecordFields::ACCESSION_KEY] ||
+        ddbj_record[DdbjRecordFields::IDENTIFIER_KEY] ||
+        ddbj_record[DdbjRecordFields::ID_KEY] ||
+        ddbj_record[DdbjRecordFields::PRIMARY_ID_KEY]
+    end
+
+    def inferred_record_type(ddbj_record)
+      DdbjRecordFields::SRA_RUN_RESOURCE_TYPE if run_record?(ddbj_record)
+    end
+  end
+end

data/lib/dratools/traversal_node.rb

@@ -0,0 +1,68 @@
+# frozen_string_literal: true
+
+module Dratools
+  # DDBJ record traversal and download leaves for tree rendering.
+  class TraversalNode
+    ROOT_RELATION = :root
+    DB_XREF_RELATION = :db_xref
+    CHILD_BIOPROJECT_RELATION = :child_bioproject
+    DOWNLOAD_RELATION = :download
+
+    attr_reader :relation, :type, :accession, :object_type, :record, :url, :error, :children,
+                :download
+
+    def initialize( # rubocop:disable Metrics/ParameterLists
+      relation: ROOT_RELATION,
+      type: nil,
+      accession: nil,
+      object_type: nil,
+      record: nil,
+      url: nil,
+      error: nil,
+      children: [],
+      download: nil
+    )
+      @relation = relation
+      @type = type
+      @accession = accession
+      @object_type = object_type
+      @record = record
+      @url = url
+      @error = error
+      @children = children
+      @download = download
+    end
+
+    def run?
+      type == DdbjRecordFields::SRA_RUN_RESOURCE_TYPE
+    end
+
+    def download?
+      relation == DOWNLOAD_RELATION
+    end
+
+    def errored?
+      !error.to_s.empty?
+    end
+
+    def run_records
+      records = run? && record ? [record] : []
+      records + children.flat_map(&:run_records)
+    end
+
+    def run_accessions
+      accessions = run? && accession ? [accession] : []
+      accessions + children.reject(&:download?).flat_map(&:run_accessions)
+    end
+
+    def downloads
+      own_downloads = download ? [download] : []
+      own_downloads + children.flat_map(&:downloads)
+    end
+
+    def errors
+      own_errors = errored? ? [error] : []
+      own_errors + children.flat_map(&:errors)
+    end
+  end
+end

data/lib/dratools/tree_renderer.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+module Dratools
+  # Renders a TraversalNode tree as terminal-friendly text.
+  class TreeRenderer
+    DEFAULT_SUMMARY_THRESHOLD = 5
+
+    def initialize(file_type: DdbjRecordFields::FILE_TYPE_SRA,
+                   summary_threshold: DEFAULT_SUMMARY_THRESHOLD)
+      @file_type = file_type
+      @summary_threshold = summary_threshold
+    end
+
+    def render(root)
+      lines = [label_for(root)]
+      render_children(root.children, prefix: '', lines: lines)
+      lines.join("\n")
+    end
+
+    private
+
+    def render_children(children, prefix:, lines:)
+      display_children = summarized_children(children)
+      display_children.each_with_index do |child, index|
+        last = index == display_children.length - 1
+        connector = last ? '└─ ' : '├─ '
+        lines << "#{prefix}#{connector}#{label_for(child)}"
+        next if child.children.empty?
+
+        child_prefix = "#{prefix}#{last ? '   ' : '│  '}"
+        render_children(child.children, prefix: child_prefix, lines: lines)
+      end
+    end
+
+    def summarized_children(children)
+      return children unless summarizable_run_group?(children)
+
+      [
+        TraversalNode.new(
+          type: DdbjRecordFields::SRA_RUN_RESOURCE_TYPE,
+          accession: "#{children.length} records",
+          children: [
+            TraversalNode.new(
+              type: summary_leaf_label(children)
+            )
+          ]
+        )
+      ]
+    end
+
+    def summary_leaf_label(children)
+      return "#{@file_type} downloads not expanded" if children.any? { |child| child.record.nil? }
+
+      "no #{@file_type} downloads"
+    end
+
+    def summarizable_run_group?(children)
+      return false if children.length <= @summary_threshold
+      return false unless children.all?(&:run?)
+
+      children.all? { |child| child.downloads.empty? && child.errors.empty? }
+    end
+
+    def label_for(node)
+      return [node.type, node.url].compact.join(' ') if node.download?
+
+      label_parts = []
+      label_parts << relation_label(node)
+      label_parts << node.type unless node.relation == TraversalNode::CHILD_BIOPROJECT_RELATION
+      label_parts << node.accession
+      label_parts << node.object_type
+      label_parts << "error: #{node.error}" if node.errored?
+      label_parts.compact.join(' ')
+    end
+
+    def relation_label(node)
+      case node.relation
+      when TraversalNode::CHILD_BIOPROJECT_RELATION
+        'childBioProject'
+      end
+    end
+  end
+end

data/lib/dratools/version.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Dratools
+  NAME = 'dratools'
+  VERSION = '0.0.1'
+end

data/lib/dratools.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+require_relative 'dratools/version'
+require_relative 'dratools/errors'
+require_relative 'dratools/config'
+require_relative 'dratools/ddbj_record_fields'
+require_relative 'dratools/byte_formatter'
+require_relative 'dratools/download_candidate'
+require_relative 'dratools/traversal_node'
+require_relative 'dratools/tree_renderer'
+require_relative 'dratools/download_candidate_builder'
+require_relative 'dratools/checksum_verifier'
+require_relative 'dratools/ddbj_resource_client'
+require_relative 'dratools/accession_resource_type_classifier'
+require_relative 'dratools/run_record_collector'
+require_relative 'dratools/accession_resolver'
+require_relative 'dratools/download_service'
+require_relative 'dratools/accession_input_collector'
+require_relative 'dratools/command_line_interface'

metadata

@@ -0,0 +1,76 @@
+--- !ruby/object:Gem::Specification
+name: dratools
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- kojix2
+bindir: bin
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies: []
+description: DDBJ Search を使って DRA/SRA ファイル URL を解決し、DDBJ から取得する小さな CLI とライブラリです。
+email:
+- 2xijok@gmail.com
+executables:
+- dratools
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.txt
+- README.md
+- bin/dratools
+- docs/design.md
+- docs/development.md
+- docs/environment.md
+- docs/usage.md
+- lib/dratools.rb
+- lib/dratools/accession_input_collector.rb
+- lib/dratools/accession_resolver.rb
+- lib/dratools/accession_resource_type_classifier.rb
+- lib/dratools/byte_formatter.rb
+- lib/dratools/checksum_verifier.rb
+- lib/dratools/command_line_interface.rb
+- lib/dratools/commands/base_command.rb
+- lib/dratools/commands/get_command.rb
+- lib/dratools/commands/meta_command.rb
+- lib/dratools/commands/probe_command.rb
+- lib/dratools/commands/runs_command.rb
+- lib/dratools/commands/size_command.rb
+- lib/dratools/commands/tree_command.rb
+- lib/dratools/commands/url_command.rb
+- lib/dratools/config.rb
+- lib/dratools/ddbj_record_fields.rb
+- lib/dratools/ddbj_resource_client.rb
+- lib/dratools/download_candidate.rb
+- lib/dratools/download_candidate_builder.rb
+- lib/dratools/download_service.rb
+- lib/dratools/errors.rb
+- lib/dratools/external_command_runner.rb
+- lib/dratools/run_record_collector.rb
+- lib/dratools/traversal_node.rb
+- lib/dratools/tree_renderer.rb
+- lib/dratools/version.rb
+homepage: https://github.com/kojix2/dratools
+licenses:
+- MIT
+metadata:
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.10
+specification_version: 4
+summary: DDBJ DRA toolkit
+test_files: []