miga-base 0.5.5.1 → 0.5.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 405d3cf6f84fa7f78f8026d1392be3fd8c32b3caea3d9e2c4c29a511ffeccb6c
-  data.tar.gz: 29108306ca8358a1155f791a4a586fce73eb9dccbbf2841869ed450064508774
+  metadata.gz: 2898bb1227f7e473f1b11d89de6cec0dea005573374c2dac19d7dcaf21143a4b
+  data.tar.gz: f6dcbfeeccf9cbcc3d8103f2c27babb47f1594f43252e131dbc67ca9ff2ef294
 SHA512:
-  metadata.gz: 84e3264d6d55ff810b6ba0ae415abac248b65ec411fde5780107c1974923c2ef7da2cd1ae86c74f36aad94af03534b1f326327f797eaf6aec1c2a20c444dfa8d
-  data.tar.gz: cff5d84655677d1eabe298478d099074c1769cb4c74a753b10e3cbd62fb9fb0ed3b4df2ccc8924799c04d8a2e6be6ac7b3720f8ccbf7d83885cf17c5537fcd97
+  metadata.gz: 42c0349e42347f2bd5bdbab6eb6dd7520335c5b3e1bb107685a80691d5ed0cc385f0a475328183db3c5e40e8e6d434495794696804d4c1c6c9c550fc0891f05a
+  data.tar.gz: 75b711267b414526af80641a8ef8cd308c0b7e7593091772fbf3ea6916da7360d3ddd3ff980df3d7e19db621731596ab284a32f7f18b05241e23606d673340ab

data/lib/miga/cli/action.rb

@@ -27,6 +27,7 @@ class MiGA::Cli::Action < MiGA::MiGA
   ##
   # Launch the sequence
   def launch
+    MiGA.DEBUG 'Cli::Action.launch'
     empty_action if cli.argv.empty?
     parse_cli
     perform

data/lib/miga/cli/action/wf.rb

@@ -120,6 +120,7 @@ module MiGA::Cli::Action::Wf
     # Define project metadata
     p = cli.load_project(:outdir, '-o')
     [:haai_p, :aai_p, :ani_p, :ess_coll].each { |i| p_metadata[i] = cli[i] }
+    p_metadata[:type] = cli[:project_type]
     transfer_metadata(p, p_metadata)
     # Download datasets
     call_cli([
@@ -138,6 +139,7 @@ module MiGA::Cli::Action::Wf
     ] + cli.files) unless cli.files.empty?
     # Define datasets metadata
     p.load
+    d_metadata[:type] = cli[:dataset_type]
     p.each_dataset { |d| transfer_metadata(d, d_metadata) }
     p
   end
@@ -165,6 +167,7 @@ module MiGA::Cli::Action::Wf
 
   def call_cli(cmd)
     cmd << '-v' if cli[:verbose]
+    MiGA::MiGA.DEBUG "Cli::Action::Wf.call_cli #{cmd}"
     MiGA::Cli.new(cmd.map(&:to_s)).launch
   end
 

data/lib/miga/cli/opt_helper.rb

@@ -155,6 +155,6 @@ module MiGA::Cli::OptHelper
   # If +sym+ is nil, +flag+ is used as Symbol
   def opt_flag(opt, flag, description, sym = nil)
     sym = flag.to_sym if sym.nil?
-    opt.on("--#{flag}", description) { |v| self[sym] = v }
+    opt.on("--#{flag.to_s.gsub('_','-')}", description) { |v| self[sym] = v }
   end
 end

data/lib/miga/daemon.rb

@@ -71,12 +71,18 @@ class MiGA::Daemon < MiGA::MiGA
   # Launches the +task+ with options +opts+ (as command-line arguments).
   # Supported tasks include: start, stop, restart, status.
   def daemon(task, opts=[])
+    MiGA.DEBUG "Daemon.daemon #{task} #{opts}"
     options = default_options
     opts.unshift(task)
     options[:ARGV] = opts
-    Daemons.run_proc("MiGA:#{project.name}", options) do
-      loop { break unless in_loop }
+    # This additional degree of separation below was introduced so the Daemons
+    # package doesn't kill the parent process in workflows.
+    pid = fork do 
+      Daemons.run_proc("MiGA:#{project.name}", options) do
+        loop { break unless in_loop }
+      end
     end
+    Process.wait pid
   end
 
   ##
@@ -321,7 +327,14 @@ class MiGA::Daemon < MiGA::MiGA
         var: %w[key value],
         alive: %w[pid],
         kill: %w[pid]
-      }.each { |k,v| runopts(k, sprintf(runopts(k), *v.map{ |i| "{{#{i}}}" })) }
+      }.each do |k,v|
+        runopts(
+          k, sprintf(
+            runopts(k).gsub(/%(\d+)\$d/, '%\\1$s'),
+            *v.map{ |i| "{{#{i}}}" }
+          )
+        )
+      end
       runopts(:format_version, 1)
     end
 end

data/lib/miga/dataset.rb

@@ -55,6 +55,7 @@ class MiGA::Dataset < MiGA::MiGA
       File.expand_path("metadata/#{name}.json", project.path),
       metadata
     ]
+    save unless File.exist? @metadata_future[0]
   end
 
   ##
@@ -66,10 +67,8 @@ class MiGA::Dataset < MiGA::MiGA
   ##
   # Save any changes you've made in the dataset.
   def save
-    if t = metadata[:tax] and n = t[:ns] and n == 'COMMUNITY'
-      self.metadata[:type] = :metagenome
-    end
-    self.metadata.save
+    MiGA.DEBUG "Dataset.metadata: #{metadata.data}"
+    metadata.save
   end
   
   ##

data/lib/miga/dataset/result.rb

@@ -116,6 +116,48 @@ module MiGA::Dataset::Result
     end
     adv
   end
+
+  ##
+  # Returns a Hash with tasks as key and status as value.
+  # See +result_status+ for possible values
+  def results_status
+    Hash[@@PREPROCESSING_TASKS.map { |task| [task, result_status(task)] }]
+  end
+
+  ##
+  # Returns the status of +task+. The status values are symbols:
+  # - ignore_inactive: the dataset is inactive
+  # - ignore_force: forced to ignore by metadata
+  # - ignore_project: incompatible project
+  # - ignore_noref: incompatible dataset, only for reference
+  # - ignore_multi: incompatible dataset, only for multi
+  # - ignore_nonmulti: incompatible dataset, only for nonmulti
+  # - ignore: incompatible dataset, unknown reason
+  # - complete: a task with registered results
+  # - pending: a task queued to be performed
+  def result_status(task)
+    if not get_result(task).nil?
+      :complete
+    elsif ignore_task?(task)
+      if not is_active?
+        :ignore_inactive
+      elsif metadata["run_#{task}"]
+        :ignore_force
+      elsif task == :taxonomy and project.metadata[:ref_project].nil?
+        :ignore_project
+      elsif @@_EXCLUDE_NOREF_TASKS_H[task] && ! is_ref?
+        :ignore_noref
+      elsif @@_ONLY_MULTI_TASKS_H[task] && ! is_multi?
+        :ignore_multi
+      elsif @@_ONLY_NONMULTI_TASKS_H[task] && ! is_nonmulti?
+        :ignore_nonmulti
+      else
+        :ignore
+      end
+    else
+      :pending
+    end
+  end
   
   ##
   # Clean-up all the stored distances, removing values for datasets no longer in

data/lib/miga/remote_dataset.rb

@@ -139,6 +139,7 @@ class MiGA::RemoteDataset < MiGA::MiGA
       rank = 'dataset' if lineage.empty? and rank.nil?
       lineage[rank] = name unless rank.nil? or rank.nil?
     end
+    MiGA.DEBUG "Got lineage: #{lineage}"
     MiGA::Taxonomy.new(lineage)
   end
 
@@ -210,6 +211,9 @@ class MiGA::RemoteDataset < MiGA::MiGA
         metadata[:is_type] = true
         metadata[:type_rel] = from_type
       end
+      metadata[:suspect] = (ncbi_asm_json_doc['exclfromrefseq'] || [])
+      metadata[:suspect] = nil if metadata[:suspect].empty?
+      MiGA.DEBUG "Got type: #{from_type}"
       metadata
     end
 

data/lib/miga/version.rb

@@ -10,7 +10,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.5, 5, 1]
+  VERSION = [0.5, 7, 2]
 
   ##
   # Nickname for the current major.minor version.
@@ -18,7 +18,7 @@ module MiGA
 
   ##
   # Date of the current gem release.
-  VERSION_DATE = Date.new(2020, 2, 4)
+  VERSION_DATE = Date.new(2020, 2, 8)
 
   ##
   # Reference of MiGA.

data/scripts/project_stats.bash

@@ -12,7 +12,7 @@ cd "$DIR"
 miga date > "miga-project.start"
 
 # Index taxonomy
-miga index_taxonomy -P "$PROJECT" -i "miga-project.taxonomy.json" --ref
+miga tax_index -P "$PROJECT" -i "miga-project.taxonomy.json" --ref --active
 
 # Index metadata
 ruby -I "$MIGA/lib" \

data/test/daemon_test.rb

@@ -79,7 +79,7 @@ class DaemonTest < Test::Unit::TestCase
     dpath = File.expand_path("daemon/MiGA:#{p.name}",p.path)
     assert(File.exist?("#{dpath}.pid"))
     out = capture_stdout { d.stop }
-    assert(out.string =~ /MiGA:#{p.name}: trying to stop process with pid \d+/)
+    assert_equal('', out.string)
     assert(!File.exist?("#{dpath}.pid"))
     assert(File.exist?("#{dpath}.output"))
     File.open("#{dpath}.output", "r") do |fh|

data/test/dataset_test.rb

@@ -50,7 +50,7 @@ class DatasetTest < Test::Unit::TestCase
     assert(!d2.is_multi?)
     assert(!d2.is_nonmulti?)
     assert_nil(d2.metadata[:type])
-    d2.metadata[:tax] = {:ns=>"COMMUNITY"}
+    d2.metadata[:type] = :metagenome
     d2.save
     assert_equal(:metagenome, d2.metadata[:type])
     assert(d2.is_multi?)

data/utils/enveomics/Pipelines/assembly.pbs/FastA.N50.pl

@@ -1 +1 @@
-utils/enveomics/Pipelines/assembly.pbs/../../Scripts/FastA.N50.pl
+../../Scripts/FastA.N50.pl

data/utils/enveomics/Pipelines/assembly.pbs/FastA.filterN.pl

@@ -1 +1 @@
-utils/enveomics/Pipelines/assembly.pbs/../../Scripts/FastA.filterN.pl
+../../Scripts/FastA.filterN.pl

data/utils/enveomics/Pipelines/assembly.pbs/FastA.length.pl

@@ -1 +1 @@
-utils/enveomics/Pipelines/assembly.pbs/../../Scripts/FastA.length.pl
+../../Scripts/FastA.length.pl

data/utils/enveomics/Pipelines/blast.pbs/FastA.split.pl

@@ -1 +1 @@
-utils/enveomics/Pipelines/blast.pbs/../../Scripts/FastA.split.pl
+../../Scripts/FastA.split.pl

data/utils/enveomics/Scripts/lib/enveomics.R

@@ -1 +1 @@
-utils/enveomics/Scripts/lib/../../enveomics.R
+../../enveomics.R

data/utils/find-medoid.R

@@ -20,18 +20,23 @@ find_medoids <- function(ani.df, out, clades) {
   dist <- enve.df2dist(ani.df, 'a', 'b', 'd', default.d = max(ani.df$d)*1.2)
   dist <- as.matrix(dist)
   cl <- read.table(clades, header = FALSE, sep = '\t', as.is = TRUE)[,1]
+  cl.s <- c()
   medoids <- c()
   for(i in cl){
     lab <- strsplit(i, ',')[[1]]
     cat('Clade of:', lab[1], '\n')
     if(length(lab) == 1) {
-      med <- lab
+      lab.s <- lab
     } else {
-      med <- lab[which.min(colSums(dist[lab, lab], na.rm = TRUE))]
+      lab.s <- lab[order(colSums(dist[lab, lab], na.rm = TRUE))]
     }
+    med <- lab.s[1]
     medoids <- c(medoids, med)
+    cl.s <- c(cl.s, paste(lab.s, collapse = ','))
   }
   write.table(medoids, out, quote = FALSE, row.names = FALSE, col.names = FALSE)
+  write.table(cl.s, paste(clades, '.sorted', sep = ''), quote = FALSE,
+    row.names = FALSE, col.names = FALSE)
 }
 
 #= Main

data/utils/index_metadata.rb

@@ -1,23 +1,24 @@
 #!/usr/bin/env ruby
 
-require "miga"
-require "sqlite3"
+require 'miga'
+require 'sqlite3'
 
 p = MiGA::Project.load(ARGV[0])
 raise "Impossible to load project: #{ARGV[0]}." if p.nil?
 
 File.unlink(ARGV[1]) if File.exist? ARGV[1]
 db = SQLite3::Database.new(ARGV[1])
-db.execute "create table metadata(" +
-  "`name` varchar(256), `field` varchar(256), `value` text)"
+db.execute 'create table metadata(' \
+  '`name` varchar(256), `field` varchar(256), `value` text)'
 
 def searchable(db, d, k, v)
-  db.execute "insert into metadata values(?,?,?)",
-    d.name, k.to_s, " #{v.to_s.downcase.gsub(/[^A-Za-z0-9\-]+/, " ")} "
+  db.execute 'insert into metadata values(?,?,?)',
+    d.name, k.to_s, " #{v.to_s.downcase.gsub(/[^A-Za-z0-9\-]+/, ' ')} "
 end
 
 p.each_dataset do |name, d|
   next unless d.is_ref?
+  next unless d.is_active?
   searchable(db, d, :name, d.name)
   d.metadata.each do |k, v|
     next if [:created, :updated].include? k

data/utils/subclade/pipeline.rb

@@ -47,6 +47,9 @@ module MiGA::SubcladeRunner::Pipeline
     dir = opts[:gsp_metric] == 'aai' ? '02.aai' : '03.ani'
     `Rscript '#{src}' ../../09.distances/#{dir}/miga-project.Rdata \
       miga-project.gsp-medoids miga-project.gsp-clades`
+    if File.exist? 'miga-project.gsp-clades.sorted'
+      File.rename 'miga-project.gsp-clades.sorted', 'miga-project.gsp-clades'
+    end
 
     # Propose clades
     ofh = File.open('miga-project.proposed-clades', 'w')

data/utils/subclades.R

@@ -113,7 +113,7 @@ subclade_clustering <- function(out_base, thr, ani.d, dist_rdata) {
   }
   write.tree(ani.ph, paste(out_base, ".nwk", sep=""))
   options(expressions=express.ori)
-  
+
   # Silhouette
   say("Silhouette")
   nn <- length(labels(ani.d))
@@ -130,13 +130,13 @@ subclade_clustering <- function(out_base, thr, ani.d, dist_rdata) {
   ds <- s.avg.z - s.neg.z - 2/(1:length(k)) - (1:length(k))/50
   if(mean(s[1,]<0)<0.75) ds[s[1,]<0] <- mean(ds) # <- k's with negative average
   top.n <- k[which.max(ds)]
-  
+
   # Classify genomes
   say("Classify => k :", top.n, "| n :", length(labels(ani.d)))
   ani.cl <- pam(ani.d, top.n, pamonce=1)
   ani.types <- ani.cl$clustering
   ani.medoids <- ani.cl$medoids
-  
+
   # Generate graphic report
   say("Graphic report")
   pdf(paste(out_base, ".pdf", sep=""), 7, 12)
@@ -149,7 +149,7 @@ subclade_clustering <- function(out_base, thr, ani.d, dist_rdata) {
 
   # Save results
   write_text_report(out_base, ani.d, ani.medoids, ani.types)
-  
+
   # Return data
   say("Cluster ready")
   return(list(
@@ -172,8 +172,8 @@ generate_empty_files <- function(out_base) {
 }
 
 write_text_report <- function(out_base, ani.d, ani.medoids, ani.types){
-  say("Text report")
-  write.table(ani.medoids, paste(out_base, "medoids", sep="."),
+  say('Text report')
+  write.table(ani.medoids, paste(out_base, 'medoids', sep='.'),
     quote=FALSE, col.names=FALSE, row.names=FALSE)
   classif <- cbind(names(ani.types), ani.types, ani.medoids[ ani.types ], NA)
   ani.d.m <- 100 - as.matrix(ani.d)*100

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 0.5.5.1
+  version: 0.5.7.2
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-02-04 00:00:00.000000000 Z
+date: 2020-02-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons
@@ -515,8 +515,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: MiGA