miga-base 1.0.5.0 → 1.0.5.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bc2d6a45a59d47e3490d1e6bbe728961abfc28002c945c0fc371b24277f7abcd
-  data.tar.gz: c20265a37c53e403858c592a72b2de801ab6ce12e48f0fe88d886cea41a94413
+  metadata.gz: d5f917644f7f3cf91b850400459b98fedfd130b002d9ef964fbbb4a0e7df446b
+  data.tar.gz: c7bed61b0725aa425fd1bb6ac69a5f45f3ee9b6d678096c91ee4d4a326283b51
 SHA512:
-  metadata.gz: 91804137814911287b5f1395e79b28e2805a0584dfb5ff234ca51a2366d17d497acf8a88ee80e18a1b8dd27a1b159ab3433471f82b389fd741825a1f5da57da5
-  data.tar.gz: 97d932379e302a22b2c0f489b5dd6cdd5efe4408c77e9cd32bb3b89efff3bc5f3787d167bc8f46720777dcc4d2efa06fb9b82f77e1903735a4b6e631e20be6f9
+  metadata.gz: db5767035a229029bdc77d7ced5ca104bce12947c7fd18b9947328aa79b0708527537c951fdef414a24ca7d1f394db5c92c4c45ade830c0114baff2fc38260d3
+  data.tar.gz: f4ce4050068ad347a35442d346a502817738082fbfb23dc3d721e0be8ee3244e83608ff0273bef57f8c3d4bb443597c95ec18f26fc218959f47ea95159842b3a

data/bin/miga-env

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+MIGA_MOD="${MIGA_HOME:-"$HOME"}/.miga_modules"
+[[ -s "$MIGA_MOD" ]] && . "$MIGA_MOD"
+"$(dirname "$0")/miga" env
+

data/lib/miga/version.rb

@@ -12,7 +12,7 @@ module MiGA
   # - String indicating release status:
   #   - rc* release candidate, not released as gem
   #   - [0-9]+ stable release, released as gem
-  VERSION = [1.0, 5, 0].freeze
+  VERSION = [1.0, 5, 4].freeze
 
   ##
   # Nickname for the current major.minor version.
@@ -20,7 +20,7 @@ module MiGA
 
   ##
   # Date of the current gem release.
-  VERSION_DATE = Date.new(2021, 8, 26)
+  VERSION_DATE = Date.new(2021, 9, 14)
 
   ##
   # References of MiGA

data/scripts/miga.bash

@@ -3,8 +3,8 @@
 ###
 # Setup environment
 set -e
-eval "$("$MIGA/bin/miga" env)"
-SCRIPT=${SCRIPT:-$(basename "$0" .bash)}
+eval "$("$MIGA/bin/miga-env")"
+SCRIPT=${SCRIPT:-"$(basename "$0" .bash)"}
 
 ###
 # Ancillary functions

data/scripts/mytaxa_scan.bash

@@ -21,7 +21,7 @@ else
     MT=$(dirname -- "$(which MyTaxa)")
     DB="$MIGA_HOME/.miga_db/AllGenomes.faa.dmnd"
     [[ -e "$DB" ]] || DB="$MT/AllGenomes.faa.dmnd"
-    if [[ ! -e "$MT/AllGenomes.faa.dmnd" ]] ; then
+    if [[ ! -e "$DB" ]] ; then
       echo "Cannot locate the database: AllGenomes.faa.dmnd:" \
             "no such file or directory" >&2
       exit 1

data/utils/enveomics/Manifest/Tasks/blasttab.json

@@ -589,7 +589,17 @@
         "|",
         { "arg": "task" },
         "--blast",
-        "/dev/stdin",
+        "-",
+        {
+          "name": "Output",
+          "opt": "--out",
+          "arg": "out_file",
+          "mandatory": true,
+          "description": [
+            "Output filtered BLAST file.",
+            "Supports compression with .gz extension."
+          ]
+        },
         {
           "opt": "--top",
           "arg": "integer",
@@ -599,19 +609,13 @@
         {
           "opt": "--sort-by",
           "arg": "select",
-          "values": ["bitscore", "evalue", "identity", "length"],
+          "values": ["bitscore", "evalue", "identity", "length", "no"],
           "default": "bitscore",
           "description": "Parameter used to detect the 'best' hits."
         },
         {
           "opt": "--quiet",
           "description": "Run quietly."
-        },
-        ">",
-        {
-          "arg": "out_file",
-          "mandatory": true,
-          "description": "Output (filtered) Tabular BLAST."
         }
       ]
     },

data/utils/enveomics/Manifest/Tasks/sequence-identity.json

@@ -536,8 +536,10 @@
     },
     {
       "task": "rbm.rb",
-      "description": ["Finds the reciprocal best matches between two sets of",
-        "sequences."],
+      "description": [
+        "Finds the reciprocal best matches between two sets of",
+        "sequences."
+      ],
       "help_arg": "--help",
       "cite":[
         ["Camacho et al, 2009, BMC Bioinf (BLAST+)",
@@ -564,11 +566,22 @@
           "mandatory": true,
           "description": "FastA file containing the genome 2."
         },
+        {
+          "name": "Output",
+          "opt": "--out",
+          "arg": "out_file",
+          "mandatory": true,
+          "description": [
+            "Reciprocal Best Matches in BLAST tabular format.",
+            "Supports compression with .gz extension."
+          ]
+        },
         {
           "name": "Nucleotides",
           "opt": "--nucl",
-          "description": ["Sequences are assumed to be nucleotides (proteins",
-            "by default)."]
+          "description": [
+            "Sequences are assumed to be nucleotides (proteins by default)."
+          ]
         },
         {
           "name": "Length",
@@ -582,8 +595,10 @@
           "opt": "--fract",
           "arg": "float",
           "default": 0.0,
-          "description": ["Minimum alignment length (as a fraction of the",
-            "query). If set, requires BLAST+ or Diamond (see Program)."]
+          "description": [
+            "Minimum alignment length (as a fraction of the query).",
+            "If set, requires BLAST+ or Diamond (see Program)."
+          ]
         },
         {
           "name": "Identity",
@@ -603,8 +618,9 @@
           "name": "Executables",
           "opt": "--bin",
           "arg": "in_dir",
-          "description": ["Directory containing the binaries of the search",
-            "program."]
+          "description": [
+            "Directory containing the binaries of the search program."
+          ]
         },
         {
           "opt": "--program",
@@ -612,9 +628,11 @@
           "values": ["blast+", "blast", "blat", "diamond"],
           "default": "blast+",
           "description": "Search program to be used. Default: blast+.",
-          "note": ["Make sure that you have installed the search program you",
+          "note": [
+            "Make sure that you have installed the search program you",
             "want to use. If you have downloaded the program, but it's not",
-            "installed, please use the Executables option above."]
+            "installed, please use the Executables option above."
+          ]
         },
         {
           "opt": "--threads",
@@ -625,12 +643,6 @@
         {
           "opt": "--quiet",
           "description": "Run quietly (no STDERR output)."
-        },
-        ">",
-        {
-          "arg": "out_file",
-          "mandatory": true,
-          "description": "Reciprocal Best Matches in BLAST tabular format."
         }
       ]
     }

data/utils/enveomics/Manifest/examples.json

@@ -26,10 +26,14 @@
     },
     {
       "task": "rbm.rb",
-      "description": ["Reciprocal Best Matches between the proteomes of the",
-        "two major HIV types (HIV-1 and HIV-2)."],
-      "values": ["hiv1.faa","hiv2.faa",null,null,null,null,null,null,null,null,
-        null,null,"hiv1-hiv2.rbm"]
+      "description": [
+        "Reciprocal Best Matches between the proteomes of the",
+        "two major HIV types (HIV-1 and HIV-2)."
+      ],
+      "values": [
+        "hiv1.faa", "hiv2.faa", "hiv1-hiv2.rbm", null, null, null, null, null,
+        null, null,null, null
+      ]
     },
     {
       "task": "ogs.mcl.rb",
@@ -83,10 +87,14 @@
     {
       "_": "== Examples of BLAST statistics and manipulation",
       "task": "BlastTab.topHits_sorted.rb",
-      "description": ["Extract the best match of metagenome-derived proteins",
-        "(from the 'A metagenome') against a Gene Ontology collection."],
-      "values": ["sort","a_mg.cds-go.blast.tsv",null,null,null,null,1,null,null,
-        null,"a_mg.cds-go.blast-bm.tsv"]
+      "description": [
+        "Extract the best match of metagenome-derived proteins",
+        "(from the 'A metagenome') against a Gene Ontology collection."
+      ],
+      "values": [
+        "sort","a_mg.cds-go.blast.tsv", null, null, null, null,
+        "a_mg.cds-go.blast-bm.tsv", 1, null, null
+      ]
     },
     {
       "task": "BlastTab.sumPerHit.pl",

data/utils/enveomics/Scripts/BlastTab.topHits_sorted.rb

@@ -1,101 +1,123 @@
 #!/usr/bin/env ruby
 
-#
-# @author:  Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
-# @update:  Dec-11-2015
-# @license: artistic license 2.0
-#
+# frozen_string_literal: true
 
-require "optparse"
+$:.push File.expand_path('../lib', __FILE__)
+require 'enveomics_rb/enveomics'
+require 'enveomics_rb/match'
+$VERSION = 1.0
 
-$opts = {n:5, sortby:"bitscore", q:false}
-$cols = {"bitscore"=>11, "evalue"=>10, "identity"=>2, "length"=>3}
-ARGV << "-h" if ARGV.size==0
+o = { n: 5, sortby: :bitscore, out: '-' }
 OptionParser.new do |opts|
-   opts.banner = "Reports the top-N best hits of a BLAST, pre-sorted by query."
-   opts.separator ""
-   opts.separator "Mandatory"
-   opts.on("-i", "--blast FILE",
-      "Path to the BLAST file."){ |v| $opts[:blast]=v }
-   opts.separator ""
-   opts.separator "Optional"
-   opts.on("-n", "--top INTEGER",
-      "Maximum number of hits to report for each query.",
-      "By default: #{$opts[:n]}"){ |v| $opts[:n]=v.to_i }
-   opts.on("-s", "--sort-by STRING",
-      "Parameter used to detect the 'best' hits.",
-      "Any of: bitscore (default), evalue, identity, length."
-      ){ |v| $opts[:sortby]=v }
-   opts.on("-q", "--quiet", "Run quietly."){ $opts[:q]=true }
-   opts.on("-h", "--help", "Display this screen") do
-      puts opts
-      exit
-   end
-   opts.separator ""
+  opts.version = $VERSION
+  Enveomics.opt_banner(
+    opts, 'Reports the top-N best hits of a BLAST, pre-sorted by query',
+    "#{File.basename($0)} -i in.tsv -o out.tsv [options]"
+  )
+  
+  opts.separator 'Mandatory'
+  opts.on(
+    '-i', '--blast FILE',
+    'Path to the BLAST file',
+    'Supports compression with .gz extension, use - for STDIN'
+  ) { |v| o[:in] = v }
+  opts.on(
+    '-o', '--out FILE',
+    'Output filtered BLAST file',
+    'Supports compression with .gz extension, use - for STDOUT (default)'
+  ) { |v| o[:out] = v }
+  opts.separator ''
+  opts.separator 'Filter Options'
+  opts.on(
+    '-n', '--top INTEGER', Integer,
+    'Maximum number of hits to report for each query',
+    "By default: #{o[:n]}"
+  ) { |v| o[:n] = v }
+  opts.on(
+    '-s', '--sort-by STRING',
+    'Parameter used to detect the "best" hits',
+    'Any of: bitscore (default), evalue, identity, length, no (pick first)'
+  ) { |v| o[:sortby] = v.to_sym }
+  opts.separator ''
+  opts.separator 'Other Options'
+  opts.on('-q', '--quiet', 'Run quietly (no STDERR output)') { $QUIET = true }
+  opts.on('-h', '--help', 'Display this screen') { puts opts; exit }
+  opts.separator ''
 end.parse!
 
-abort "-i/--blast is mandatory." if $opts[:blast].nil?
-abort "Unrecognized value for -s/--sortby: #{$opts[:sortby]}." if
-   $cols[ $opts[:sortby] ].nil?
-
-class Hit
-   attr_reader :blast_line
-   def initialize(blast_line)
-      @blast_line = blast_line.chomp.split(/\t/)
-   end
-   def col(i)
-      @blast_line[i]
-   end
-   def <=>(other)
-      ans = self.col( $cols[ $opts[:sortby] ] ).to_f <=> other.col( $cols[ $opts[:sortby] ] ).to_f
-      ans = ans * -1 unless $opts[:sortby] == "evalue"
-      return ans
-   end
-   def to_s
-      @blast_line.join("\t")
-   end
+raise Enveomics::OptionError.new('-i is mandatory') if o[:in].nil?
+unless o[:sortby] == :no || Enveomics::Match.column(o[:sortby])
+  raise Enveomics::OptionError.new("Unrecognized value for -s: #{o[:sortby]}")
 end
 
-class HitSet
-   attr_reader :query, :hits
-   def initialize
-      @hits = []
-      @query = nil
-   end
-   def <<(hit)
-      @query = hit.col(0) if @query.nil?
-      raise "Inconsistent query, expecting #{self.query}" unless
-	 self.query == hit.col(0)
-      @hits << hit
-   end
-   def empty?
-      self.hits.length == 0
-   end
-   def filter!
-      @hits.sort!
-      @hits.slice!($opts[:n], @hits.length)
-   end
-   def to_s
-      @hits.join("\n")
-   end
+class Enveomics::Match
+  attr_accessor :sortby
+
+  def <=>(other)
+    ans = send(sortby) <=> other.send(sortby)
+    sortby == :evalue ? ans : ans * -1
+  end
 end
 
-$stderr.puts "Parsing BLAST." unless $opts[:q]
-fh = File.open $opts[:blast], "r"
-hs = HitSet.new
-while ln=fh.gets
-   hit = Hit.new( ln )
-   if hs.query != hit.col(0)
-      hs.filter!
-      puts hs unless hs.empty?
-      hs = HitSet.new
-      $stderr.print "Parsing line #{$.}... \r" unless $opts[:q]
-   end
-   hs << hit
+class Enveomics::MatchSet
+  attr_reader :query, :hits, :sortby
+
+  def initialize(sortby)
+    @hits = []
+    @query = nil
+    @sortby = sortby
+  end
+
+  def <<(hit)
+    @query ||= hit.qry
+    unless query == hit.qry
+      raise "Inconsistent query, expecting #{query}"
+    end
+
+    @hits << hit.tap { |i| i.sortby = sortby }
+  end
+
+  def empty?
+    hits.empty?
+  end
+
+  def filter!(n)
+    @hits.sort! unless sortby == :no
+    @hits.slice!(n, @hits.length)
+  end
+
+  def to_s
+    hits.join("\n")
+  end
 end
-$stderr.print "Parsed #{$.} lines.   \n" unless $opts[:q]
-fh.close
 
-hs.filter!
-puts hs unless hs.empty?
+begin
+  ifh = reader(o[:in])
+  ofh = writer(o[:out])
+
+  say 'Parsing BLAST'
+  hs = Enveomics::MatchSet.new(o[:sortby])
+  lno = 0
+  ifh.each do |ln|
+    lno += 1
+    hit = Enveomics::Match.new(ln)
+    if hs.query != hit.qry
+      hs.filter! o[:n]
+      ofh.puts hs unless hs.empty?
+      hs = Enveomics::MatchSet.new(o[:sortby])
+      say_inline("Parsing line #{lno}... \r")
+    end
+    hs << hit
+  end
+  say("Parsed #{lno} lines    ")
+  ifh.close
+
+  hs.filter! o[:n]
+  ofh.puts hs unless hs.empty?
+  ofh.close
+rescue => err
+  $stderr.puts "Exception: #{err}\n\n"
+  err.backtrace.reverse.each { |l| $stderr.puts "DEBUG: %s\n" % l }
+  err
+end
 

data/utils/enveomics/Scripts/aai.rb

@@ -316,6 +316,7 @@ Dir.mktmpdir do |dir|
   end
   res = File.open(o[:res], "w") unless o[:res].nil?
   rbm = File.open(o[:rbm], "w") unless o[:rbm].nil?
+  sqlite_db.execute('BEGIN TRANSACTION') unless o[:sqlite3].nil?
   [1,2].each do |i|
     qry_seen = []
     q = "#{dir}/seq#{i}.fa"
@@ -413,6 +414,7 @@ Dir.mktmpdir do |dir|
       o[:sqlite3].nil?
     puts id2/n2 if o[:auto]
   end
+  sqlite_db.execute('COMMIT') unless o[:sqlite3].nil?
   res.close unless o[:res].nil?
   fo.close unless o[:out].nil?
 end

data/utils/enveomics/Scripts/lib/enveomics_rb/match.rb

@@ -4,44 +4,69 @@ module Enveomics
   # A simple object representing a sequence match from a search engine
   # supporting tabular BLAST output
   class Match
-    attr :row
+    class << self
+      def column_types
+        {
+          qseqid: String,    sseqid: String,    pident: Float,
+          length: Integer,   mismatch: Integer, gapopen: Integer,
+          q_start: Integer,  q_end: Integer,    s_start: Integer,
+          s_end: Integer,    evalue: Float,     bitscore: Float,
+          # Non-standard (but frequently used in Enveomics Collection):
+          qry_len: Integer,  sbj_len: Integer
+        }
+      end
 
-    ##
-    # Initialize Enveomics::Match object from a tabular blast line String +ln+
-    def initialize(ln)
-      @row = ln.chomp.split("\t")
-    end
+      def column_type(sym)
+        column_types[colname(sym)]
+      end
 
-    def qry
-      row[0]
-    end
+      def to_column_type(sym, value)
+        case column_type(sym).to_s
+        when 'String' ; value.to_s
+        when 'Float'  ; value.to_f
+        when 'Integer'; value.to_i
+        end
+      end
 
-    def sbj
-      row[1]
-    end
+      def columns
+        column_types.keys
+      end
 
-    def id
-      @id ||= row[2].to_f
-    end
+      def column(sym)
+        columns.index(colname(sym))
+      end
 
-    def len
-      @len ||= row[3].to_i
-    end
+      def colsynonyms
+        {
+          qry: :qseqid, sbj: :sseqid,
+          id: :pident, len: :length, score: :bitscore
+        }
+      end
 
-    def evalue
-      @evalue ||= row[9].to_f
-    end
+      def colnames
+        columns + colsynonyms.keys
+      end
 
-    def score
-      @score ||= row[10].to_f
+      def colname(sym)
+        s = sym.to_sym
+        column_types[s] ? s : colsynonyms[s]
+      end
     end
 
-    def qry_len
-      @qry_len ||= row[12].to_i
+    ####--- Instance Level ---###
+
+    attr :row
+
+    ##
+    # Initialize Enveomics::Match object from a tabular blast line String +ln+
+    def initialize(ln)
+      @row = ln.chomp.split("\t")
     end
 
-    def sbj_len
-      @sbj_len ||= row[13].to_i
+    colnames.each do |sym|
+      define_method sym do
+        self.class.to_column_type(sym, row[self.class.column(sym)])
+      end
     end
 
     def qry_fract

data/utils/enveomics/Scripts/lib/enveomics_rb/utils.rb

@@ -18,10 +18,11 @@ def use(gems, mandatory = true)
 end
 
 def say(*msg)
-  return if $QUIET ||= false
+  $stderr.puts('[%s] %s' % [Time.now, msg.join('')]) unless $QUIET ||= false
+end
 
-  o = '[%s] %s' % [Time.now, msg.join('')]
-  $stderr.puts(o)
+def say_inline(*msg)
+  $stderr.print('[%s] %s' % [Time.now, msg.join('')]) unless $QUIET ||= false
 end
 
 ##

data/utils/enveomics/Scripts/rbm.rb

@@ -2,18 +2,19 @@
 
 # frozen_string_literal: true
 
-$VERSION = 1.0
+$VERSION = 1.01
 $:.push File.expand_path('../lib', __FILE__)
 require 'enveomics_rb/rbm'
 require 'tmpdir'
 
 bms_dummy = Enveomics::RBM.new('1', '2').bms1
-o = { q: false }
+o = { q: false, out: '-' }
 %i[thr len id fract score bin program nucl].each do |k|
   o[k] = bms_dummy.opt(k)
 end
 
 OptionParser.new do |opts|
+  opts.version = $VERSION
   cmd = File.basename($0)
   opts.banner = <<~BANNER
 
@@ -34,6 +35,11 @@ OptionParser.new do |opts|
     '-2', '--seq2 FILE',
     'Path to the FastA file containing the set 2'
   ) { |v| o[:seq2] = v }
+  opts.on(
+    '-o', '--out FILE',
+    'Reciprocal Best Matches in BLAST tabular format.',
+    'Supports compression with .gz extension, use - for STDOUT (default)'
+  ) { |v| o[:out] = v }
   opts.separator ''
   opts.separator 'Search Options'
   opts.on(
@@ -80,7 +86,7 @@ OptionParser.new do |opts|
   ) { |v| o[:thr] = v }
   opts.separator ''
   opts.separator 'Other Options'
-  opts.on('-q', '--quiet', 'Run quietly (no STDERR output)') { o[:q] = true }
+  opts.on('-q', '--quiet', 'Run quietly (no STDERR output)') { $QUIET = true }
   opts.on('-h', '--help', 'Display this screen') { puts opts ; exit }
   opts.separator ''
 end.parse!
@@ -90,10 +96,12 @@ raise Enveomics::OptionError.new('-2 is mandatory') if o[:seq2].nil?
 raise Enveomics::OptionError.new(
   'Argument -f/--fract requires -p blast+ or -p diamond'
 ) if o[:fract] > 0.0 && !%i[blast+ diamond].include?(o[:program])
-$QUIET = o[:q]
 
 rbm = Enveomics::RBM.new(o[:seq1], o[:seq2], o)
-rbm.each { |bm| puts bm.to_s }
+ofh = writer(o[:out])
+rbm.each { |bm| ofh.puts bm.to_s }
+ofh.close
+
 say('Forward Best Matches: ', rbm.bms1.count)
 say('Reverse Best Matches: ', rbm.bms2.count)
 say('Reciprocal Best Matches: ', rbm.count)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 1.0.5.0
+  version: 1.0.5.4
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-08-26 00:00:00.000000000 Z
+date: 2021-09-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons
@@ -107,6 +107,7 @@ files:
 - README.md
 - Rakefile
 - bin/miga
+- bin/miga-env
 - lib/miga.rb
 - lib/miga/_data/aai-intax-blast.tsv.gz
 - lib/miga/_data/aai-intax-diamond.tsv.gz