bioroebe 0.10.80

data/lib/bioroebe/taxonomy/taxonomy.rb

@@ -0,0 +1,521 @@
+#!/usr/bin/ruby -w
+# Encoding: UTF-8
+# frozen_string_literal: true
+# =========================================================================== #
+# === Bioroebe::Taxonomy
+#
+# The explanation for the format was obtained from this file:
+#
+#   ftp://ftp.ncbi.nih.gov/pub/taxonomy/taxdump_readme.txt
+#
+# =========================================================================== #
+module Bioroebe
+
+module Taxonomy
+
+  require 'bioroebe/taxonomy/class_methods.rb'
+  require 'bioroebe/taxonomy/chart.rb'
+  require 'bioroebe/taxonomy/node.rb'
+
+  require 'bioroebe/toplevel_methods/databases.rb'
+  require 'bioroebe/toplevel_methods/file_and_directory_related_actions.rb'
+  require 'bioroebe/toplevel_methods/time_and_date.rb'
+
+  begin
+    require 'roebe/sql_paradise'
+    include Roebe::SqlParadise::Commands # This derives from sql_paradise.
+  rescue LoadError; end
+
+  # ========================================================================= #
+  # Add an alias towards puts, unless we use the Colours module.
+  # ========================================================================= #
+  alias e puts unless Object.const_defined? :Colours
+
+  require 'bioroebe/requires/require_encoding.rb'
+  # ========================================================================= #
+  # === to_utf
+  # ========================================================================= #
+  def to_utf(i)
+    ::Bioroebe.to_utf(i)
+  end; alias sanitize to_utf # === sanitize
+
+  # ========================================================================= #
+  # === test
+  # ========================================================================= #
+  def test(use_this_url = URL1)
+    e 'Opening URL at "'+sfancy(use_this_url)+'"'
+    data = File.readlines(DIVISION)
+    return data
+  end
+
+  # ========================================================================= #
+  # === split                                                     (split tag)
+  #
+  # Split up the input on the default delimiter.
+  # ========================================================================= #
+  def split(
+      i = citations?
+    )
+    if i.is_a? Array
+      return i.map {|entry| entry.split(MAIN_DELIMITER) }
+    else
+      begin
+        i = sanitize(i)
+        if i.include?(TOKEN) and ! i.include?("\t")
+          i = i.split(TOKEN)
+        else
+          return i.split(MAIN_DELIMITER)
+        end
+      rescue Exception => error
+        pp error
+        e 'The string that failed was '+i
+        $error = i
+        return
+      end
+    end
+  end
+
+  # ========================================================================= #
+  # === pad
+  #
+  # The input is something like:
+  #
+  #   1457406 | Bionia Mart. ex Benth., 1837 | | authority |
+  #
+  # but it could also be an array.
+  # ========================================================================= #
+  def pad(i)
+    if i.is_a? String
+      i = i.chomp # Newlines aren't really useful - let's eliminate them.
+      if i.include?(TOKEN)
+        splitted = tokenize(i) # Now splitted input.
+        # Next, pad sql.
+        # splitted = splitted.map {|entry| pad_sql(entry) }
+        # ^^^ This may lead to problems however.
+        i = splitted
+      end
+      # Next, we will get rid of "'" characters.
+      i = i.delete("'") if i.include? "'"
+    end
+    i = i.join("','") if i.is_a? Array
+    return i
+  end
+
+  # ========================================================================= #
+  # === generate_html_links_for
+  #
+  # Input to this method should be an array of taxonomic IDs.
+  # ========================================================================= #
+  def generate_html_links_for(i)
+    base_url = 'https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=Info&id='
+    if i.is_a? Array
+      i.each {|entry| generate_html_links_for(entry) }
+    else
+      e base_url+i.to_s
+    end
+  end
+
+  # ========================================================================= #
+  # === clean
+  #
+  # Clean the input string from delimiters. Input can be Array or String.
+  # ========================================================================= #
+  def clean(i)
+    if i.is_a? Array
+      return i.map {|entry| clean(entry) }
+    else
+      return i.gsub(/#{MAIN_DELIMITER}/, ' ').chomp.strip # Added .chomp at 04.02.2014
+    end
+  end; alias remove_delimiters clean # === remove_delimiters
+
+  # ========================================================================= #
+  # === nodes?
+  #
+  # Use this method in order to obtain the nodes.dmp dataset.
+  #
+  # Nodes are "taxonomic nodes".
+  #
+  # The description for each node includes the following fields:
+  #
+  #   taxid        - node id in GenBank taxonomy database
+  #   parent taxid - parent node id in GenBank taxonomy database
+  #   rank         - rank of this node (superkingdom, kingdom, ...)
+  #   embl code    - locus-name prefix; not unique
+  #   division id  - see division.dmp file
+  #   inherited div flag (1 or 0) - 1 if node inherits division from parent
+  #   genetic code id - see gencode.dmp file
+  #   inherited GC flag (1 or 0) - 1 if node inherits genetic code from parent
+  #   mitochondrial genetic code id - see gencode.dmp file
+  #   inherited MGC flag (1 or 0) - 1 if node inherits mitochondrial gencode from parent
+  #   GenBank hidden flag (1 or 0) - 1 if name is suppressed in GenBank entry lineage
+  #   hidden subtree root flag(1 or 0) - 1 if this subtree has no sequence data yet
+  #   comments - free-text comments and citations
+  #
+  # ========================================================================= #
+  def nodes?(i = NODES)
+    Taxonomy.load(i)
+  end
+
+  # ========================================================================= #
+  # === search_id
+  #
+  # Search the Taxonomic ID here.
+  # ========================================================================= #
+  def search_id(i = '7460')
+    e BASE_URL+'id='+i.to_s
+  end; alias search search_id # === search
+       alias id?    search_id # === id?
+
+  # ========================================================================= #
+  # === names?
+  #
+  # Obtain the names.dmp dataset, which are "Taxonomy names".
+  #
+  # Four IDs are known for this set:
+  #
+  #   taxid
+  #   name_txt
+  #   unique name
+  #   name class
+  #
+  # ========================================================================= #
+  def names?(i = NAMES)
+    Taxonomy.load(i)
+  end
+
+  # ========================================================================= #
+  # === division?
+  #
+  # Obtain the division.dmp dataset.
+  #
+  # The Divisions file has these fields:
+  #
+  #   division id -- taxonomy database division id
+  #   division cde -- GenBank division code (three characters)
+  #   division name -- e.g. BCT, PLN, VRT, MAM, PRI...
+  #
+  # Comments.
+  # ========================================================================= #
+  def division?
+    Taxonomy.load(DIVISION)
+  end
+
+  # ========================================================================= #
+  # === gencode?
+  #
+  # Obtain gencode.dmp dataset, "genetic codes" file.
+  #
+  #   genetic code id -- GenBank genetic code id
+  #   abbreviation    -- genetic code name abbreviation
+  #   name            -- genetic code name
+  #   cde             -- translation table for this genetic code
+  #   starts          -- start codons for this genetic code
+  #
+  # ========================================================================= #
+  def gencode?
+    Taxonomy.load(GENCODE)
+  end
+
+  # ========================================================================= #
+  # === delnodes?
+  #
+  # Obtain the delnodes.dmp dataset. This one has deleted nodes - nodes that
+  # existed but were deleted.
+  # ========================================================================= #
+  def delnodes?
+    Taxonomy.load(DELNODES)
+  end
+
+  # ========================================================================= #
+  # === merged?
+  #
+  # Obtain information from merged.dmp.
+  #
+  # Merged nodes file fields has these ids:
+  #
+  #   old_taxid -- id of nodes which has been merged
+  #   new_taxid -- id of nodes which is result of merging
+  #
+  # ========================================================================= #
+  def merged?
+    Taxonomy.load(MERGED)
+  end
+
+  # ========================================================================= #
+  # === citations?
+  #
+  # Obtain the citations.dmp dataset through this method.
+  #
+  # This dataset includes the following keys:
+  #
+  #   cit_id, cit_key, pubmed_id,
+  #   medline_id, url,
+  #   text, taxid_list
+  #
+  # ========================================================================= #
+  def citations?
+    Taxonomy.load(CITATIONS)
+  end
+
+  # ========================================================================= #
+  # === array_size?
+  #
+  # Throwaway method to give us back all text entries from a citation base.
+  # ========================================================================= #
+  def array_size?(array_input = citations?)
+    return array_input.map {|entry| split(entry)[5].size }
+  end
+
+  # ========================================================================= #
+  # === return_full_lineage_of
+  #
+  # This method will return an array (or nil) with the full lineage of the
+  # given input (which should be a Taxonomic id).
+  #
+  # The logic for this method is as follows:
+  #
+  # - Given an arbitrary tax ID as input, we will keep on searching the
+  #   postgresql database for parent_id entries, until we eventually reach
+  #   input number 1, which is the mother of all IDs. So when we have 1
+  #   finally, we can stop, and return the result (the array in question).
+  #
+  # The full lineage is thus given as part of the returned array. The
+  # format is to not only return the ID but to also return the 
+  # scientific name. In other words, our returned array will have this
+  # format:
+  #
+  #   [parent_id, scientific_name]
+  #
+  # ========================================================================= #
+  def return_full_lineage_of(i = nil)
+    if i
+      array = []
+      array << [ i, get_scientific_name_of(i) ]
+      rescue_counter = 0
+      loop {
+        id = array[-1][0]
+        parent_id       = get_parent_id_of(id)
+        scientific_name = get_scientific_name_of(parent_id)
+        array << [ parent_id, scientific_name ]
+        rescue_counter += 1
+        break if array[-1][0].to_i == 1
+        if rescue_counter > 50
+          e 'It seems as if something is not working properly here. We reached'
+          e 'a count of 50 now, without finding a parent id (which should be 1).'
+          e 'It is quite unlikely that a lineage will have more than 50 subentries'
+          e 'so we will now break out of the loop.'
+          return array
+        end
+      }
+      return array
+    else
+      e 'Missing input. Please provide an ID (a number, like 6).'
+      return nil
+    end
+  end
+
+  # ========================================================================= #
+  # === get_parent_id_of
+  #
+  # This method is similar to the method above, but we will fetch the
+  # parent id instead.
+  # ========================================================================= #
+  def get_parent_id_of(i)
+    if i.to_s.empty?
+      e 'No valid input was given to us (in method '+__method__.to_s+').'
+    end
+    cmd = 'SELECT parent_taxid FROM nodes WHERE taxid='+i.to_s+' limit 3;'
+    result = run_sql_query(cmd) # More verbose here.
+    result = run_sql_query(cmd, true, :tuples).strip
+    return result
+  end
+
+  # ========================================================================= #
+  # === get_scientific_name_of
+  #
+  # Get the scientific name here. The input should be a taxid.
+  #
+  # Usage example:
+  #
+  #   get_scientific_name_of 333
+  #
+  # ========================================================================= #
+  def get_scientific_name_of(taxid)
+    _ = 'SELECT name_txt FROM names where taxid='+taxid.to_s+
+        ' AND name_class=\'scientific name\' LIMIT 3;'
+    result = run_sql_query(_)
+    # _ << 'SELECT name_txt FROM names where taxid='+taxid.to_s+' LIMIT 3;"'
+    result = run_sql_query(_, true, ' --tuples-only').strip
+    return result
+  end; alias get_scientific_name get_scientific_name_of # === get_scientific_name
+
+  # ========================================================================= #
+  # === get_id_of
+  #
+  # Use this method to query the database for a specific ID.
+  # For this to work, the database must have the ids.
+  # ========================================================================= #
+  def get_id_of(
+      id = 9606,
+      be_verbose = true
+    )
+    id = id.to_s
+    run_sql_query(
+      'SELECT * from names WHERE taxid='+id+' AND name_class=\'scientific name\' LIMIT 3;"',
+      be_verbose
+    )
+    result = run_sql_query(
+      'SELECT taxid from names WHERE taxid='+id+' AND name_class=\'scientific name\' LIMIT 3;"',
+      :silent, :tuples
+    ).strip
+    return result
+  end
+
+  # ========================================================================= #
+  # === find_highest_entries_in_sql
+  #
+  # This method will find the highest entries in the sql database. This
+  # may be useful if we wish to optimize the database (i.e. find the
+  # best varchar attribute in question).
+  # ========================================================================= #
+  def find_highest_entries_in_sql
+    e 'We will find the various highest entries in the sql files.'
+    this_file = TAXONOMY_DIR+'names.dmp'
+    e 'Starting with '+sfile(this_file)+' first:'
+    e 'Names has 4 entries - we are interested in all of them.'
+    e this_file
+
+    max_taxid       = 0
+    max_name_txt    = 0
+    max_unique_name = 0
+    max_name_class  = 0
+
+    e 'Please be patient, this may take a while ...'
+    readlines(this_file).each {|entry|
+      _ = split_at_tabulator(entry)
+
+      if _[0].to_s.size > max_taxid
+        max_taxid = _[0].to_s.size.to_i
+      end
+      if _[1].to_s.size > max_name_txt
+        max_name_txt = _[1].to_s.size.to_i
+      end
+      if _[2].to_s.size > max_unique_name
+        max_unique_name = _[2].to_s.size.to_i
+      end
+      if _[3].to_s.size > max_name_class
+        max_name_class = _[3].to_s.size.to_i
+      end
+    }
+
+    max_taxid = '%05s' % max_taxid
+    e '  max_taxid       is '+sfancy(max_taxid.to_s)+' characters long (should be an int anyway).'
+    e '  max_name_txt    is '+sfancy('%05s' % max_name_txt.to_s)+' characters long.'
+    e '  max_unique_name is '+sfancy('%05s' % max_unique_name.to_s)+' characters long.'
+    e '  max_name_class  is '+sfancy('%05s' % max_name_class.to_s)+' characters long.'
+
+    # Next, we will work on nodes.dmp:
+    this_file = TAXONOMY_DIR+'nodes.dmp'
+    e 'Now working on '+sfile(this_file)+' first:'
+    e 'Nodes has 3 relevant entries - taxid, parent_taxid and rank (but in total it has 13 entries)'
+    e this_file
+
+    max_taxid         = 0
+    max_parent_taxid  = 0
+    max_rank          = 0
+
+    e 'Please be patient, this may take a while ...'
+    readlines(this_file).each {|entry|
+      _ = split_at_tabulator(entry)
+
+      if _[0].to_s.size > max_taxid
+        max_taxid = _[0].to_s.size.to_i
+      end
+      if _[1].to_s.size > max_parent_taxid
+        max_parent_taxid = _[1].to_s.size.to_i
+      end
+      if _[2].to_s.size > max_rank
+        max_rank = _[2].to_s.size.to_i
+      end
+    }
+
+    e '  max_taxid        is '+
+      sfancy('%05s' % max_taxid.to_s)+' characters long.'
+    e '  max_parent_taxid is '+
+      sfancy('%05s' % max_parent_taxid.to_s)+' characters long.'
+    e '  max_rank         is '+
+      sfancy('%05s' % max_rank.to_s)+' characters long.'
+  end
+
+  # ========================================================================= #
+  # === verify_proper_sql_structures                              (debug tag)
+  #
+  # We will try to verify that the SQL commands are accurate.
+  #
+  # Can also be called by issuing this:
+  #
+  #   Taxonomy.verify_proper_sql_structures
+  #
+  # ========================================================================= #
+  def verify_proper_sql_structures
+    n_chars_to_show = 975
+    files = [ NODES_SQL, NAMES_SQL ]
+    e 'We will now attempt to verify that the SQL structure is proper.'
+    e 'We have these '+sfancy(files.size.to_s)+' .sql files.'
+    pp files
+    files.each {|entry|
+      if File.exist? entry
+        size = File.size(entry).to_s
+        chunk = File.read(entry)[0..n_chars_to_show]
+        e 'File '+sfile(entry)+' (Filesize: '+sfancy(size)+
+          ' KB) has this content '+
+          '(showing up to '+simp(n_chars_to_show)+' chars):'
+        cliner { e chunk }; e
+      else
+        e 'We can not verify the sql structure because the file'
+        e 'at `'+sfile(entry)+'` does not exist.'
+      end
+    }
+  end
+
+  # ========================================================================= #
+  # === return_current_hours_minutes_seconds
+  # ========================================================================= #
+  def return_current_hours_minutes_seconds
+    ::Bioroebe.return_current_hours_minutes_seconds
+  end
+
+  # ========================================================================= #
+  # === Taxonomy.show_current_time_and_date
+  # ========================================================================= #
+  def self.show_current_time_and_date
+    if be_verbose?
+      e 'The current time is '+sfancy(return_current_hours_minutes_seconds)+
+        ', and today is the '+simp(return_current_day_month_year)+'.'
+    end
+  end
+
+  # ========================================================================= #
+  # === remove_delimiter
+  #
+  # Get rid of the '|' token.
+  # ========================================================================= #
+  def remove_delimiter(i)
+    return i.delete(TOKEN)
+  end
+
+  # ========================================================================= #
+  # === show_current_time_and_date
+  # ========================================================================= #
+  def show_current_time_and_date
+    Taxonomy.show_current_time_and_date
+  end
+
+  extend self
+
+end; end
+
+if __FILE__ == $PROGRAM_NAME
+  require 'bioroebe'
+  Bioroebe::Taxonomy.update
+end

data/lib/bioroebe/toplevel_methods/ad_hoc_task.rb

@@ -0,0 +1,56 @@
+#!/usr/bin/ruby -w
+# Encoding: UTF-8
+# frozen_string_literal: true
+# =========================================================================== #
+# require 'bioroebe/toplevel_methods/ad_hoc_task.rb'
+# =========================================================================== #
+module Bioroebe
+
+  require 'bioroebe/toplevel_methods/e.rb'
+  require 'bioroebe/toplevel_methods/file_and_directory_related_actions.rb'
+
+  # ========================================================================= #
+  # === Bioroebe.ad_hoc_task
+  #
+  # This method can be used to specifically run an "ad-hoc" task.
+  #
+  # An ad-hoc task is something that we just quickly "hack" together,
+  # in order to solve some existing bioinformatics-related problem.
+  #
+  # Presently, in May 2021, this was for a university course that
+  # required us to work with MEGA X and compare different proteins
+  # from a phylogenetics point of view.
+  # ========================================================================= #
+  def self.ad_hoc_task(
+      this_file = '/root/Bioroebe/table_ids.md'
+    )
+    require 'bioroebe/fasta_and_fastq/download_fasta.rb'
+    require 'bioroebe/fasta_and_fastq/simplify_fasta_header/simplify_fasta_header.rb'
+    if this_file.is_a? Array
+      this_file = this_file.join(' ')
+    end
+    cd ::Bioroebe.log_dir? # Make sure we are in the log-directory.
+    e 'Now downloading some FASTA files, based on this file: '+
+      this_file
+    # ======================================================================= #
+    # (1) Download the remote FASTA dataset
+    # ======================================================================= #
+    download_fasta this_file
+    # ======================================================================= #
+    # (2) cd into the fasta directory
+    # ======================================================================= #
+    cd ::Bioroebe.log_dir?+'fasta/'
+    # ======================================================================= #
+    # (3) batch rename all .fasta files next via simplify-fasta-header.
+    # ======================================================================= #
+    all_files = Dir['*.fasta']
+    all_files.each {|this_fasta_file|
+      Bioroebe.overwrite_fasta_header(this_fasta_file)
+    }
+  end
+
+end
+
+if __FILE__ == $PROGRAM_NAME
+  Bioroebe.ad_hoc_task
+end # adhoctask