miga-base 0.3.1.3 → 0.3.1.4

data/lib/miga/project/base.rb

@@ -0,0 +1,99 @@
+# @package MiGA
+# @license Artistic-2.0
+
+class MiGA::Project < MiGA::MiGA
+
+  class << self
+    ##
+    # Does the project at +path+ exist?
+    def exist?(path)
+      Dir.exist?(path) and File.exist?("#{path}/miga.project.json")
+    end
+
+    ##
+    # Load the project at +path+. Returns MiGA::Project if project exists, nil
+    # otherwise.
+    def load(path)
+      return nil unless exist? path
+      new path
+    end
+
+    def INCLADE_TASKS ; @@INCLADE_TASKS ; end
+    def DISTANCE_TASKS ; @@DISTANCE_TASKS ; end
+    def KNOWN_TYPES ; @@KNOWN_TYPES ; end
+    def RESULT_DIRS ; @@RESULT_DIRS ; end
+
+  end
+
+end
+
+module MiGA::Project::Base
+
+  ##
+  # Top-level folders inside a project.
+  @@FOLDERS = %w[data metadata daemon]
+
+  ##
+  # Folders for results.
+  @@DATA_FOLDERS = %w[
+    01.raw_reads 02.trimmed_reads 03.read_quality 04.trimmed_fasta
+    05.assembly 06.cds
+    07.annotation 07.annotation/01.function 07.annotation/02.taxonomy
+    07.annotation/01.function/01.essential
+    07.annotation/01.function/02.ssu
+    07.annotation/02.taxonomy/01.mytaxa
+    07.annotation/03.qa 07.annotation/03.qa/01.checkm
+    07.annotation/03.qa/02.mytaxa_scan
+    08.mapping 08.mapping/01.read-ctg 08.mapping/02.read-gene
+    09.distances 09.distances/01.haai 09.distances/02.aai
+    09.distances/03.ani 09.distances/04.ssu 09.distances/05.taxonomy
+    10.clades 10.clades/01.find 10.clades/02.ani 10.clades/03.ogs
+    10.clades/04.phylogeny 10.clades/04.phylogeny/01.essential
+    10.clades/04.phylogeny/02.core 10.clades/05.metadata
+    90.stats
+  ]
+
+  ##
+  # Directories containing the results from project-wide tasks.
+  @@RESULT_DIRS = {
+    project_stats: "90.stats",
+    # Distances
+    haai_distances: "09.distances/01.haai",
+    aai_distances: "09.distances/02.aai",
+    ani_distances: "09.distances/03.ani",
+    #ssu_distances: "09.distances/04.ssu",
+    # Clade identification
+    clade_finding: "10.clades/01.find",
+    # Clade analysis
+    subclades: "10.clades/02.ani",
+    ogs: "10.clades/03.ogs"
+    #ess_phylogeny: "10.clades/04.phylogeny/01.essential",
+    #core_phylogeny: "10.clades/04.phylogeny/02.core",
+    #clade_metadata: "10.clades/05.metadata"
+  }
+
+  ##
+  # Supported types of projects.
+  @@KNOWN_TYPES = {
+    mixed: {
+      description: "Mixed collection of genomes, metagenomes, and viromes.",
+      single: true, multi: true},
+    genomes: {description: "Collection of genomes.",
+      single: true, multi: false},
+    clade: {description: "Collection of closely-related genomes (ANI >= 90%).",
+      single: true, multi: false},
+    metagenomes: {description: "Collection of metagenomes and/or viromes.",
+      single: false, multi: true}
+  }
+
+  ##
+  # Project-wide distance estimations.
+  @@DISTANCE_TASKS = [:project_stats,
+    :haai_distances, :aai_distances, :ani_distances, :clade_finding]
+  
+  ##
+  # Project-wide tasks for :clade projects.
+  @@INCLADE_TASKS = [:subclades, :ogs]
+  
+end
+

data/lib/miga/project/dataset.rb

@@ -0,0 +1,148 @@
+# @package MiGA
+# @license Artistic-2.0
+
+##
+# Helper module including specific functions handle datasets.
+module MiGA::Project::Dataset
+  
+
+  ##
+  # Returns Array of MiGA::Dataset.
+  def datasets
+    metadata[:datasets].map{ |name| dataset(name) }
+  end
+
+  ##
+  # Returns Array of String (without evaluating dataset objects).
+  def dataset_names
+    metadata[:datasets]
+  end
+  
+  ##
+  # Returns MiGA::Dataset.
+  def dataset(name)
+    name = name.miga_name
+    return nil unless MiGA::Dataset.exist?(self, name)
+    @datasets ||= {}
+    @datasets[name] ||= MiGA::Dataset.new(self, name)
+    @datasets[name]
+  end
+
+  ##
+  # Iterate through datasets, with one or two variables passed to +blk+.
+  # If one, the dataset MiGA::Dataset object is passed. If two, the name and
+  # the dataset object are passed.
+  def each_dataset(&blk)
+    metadata[:datasets].each do |name|
+      if blk.arity == 1
+        blk.call(dataset(name))
+      else
+        blk.call(name, dataset(name))
+      end
+    end
+  end
+  
+  ##
+  # Add dataset identified by +name+ and return MiGA::Dataset.
+  def add_dataset(name)
+    unless metadata[:datasets].include? name
+      MiGA::Dataset.new(self, name)
+      @metadata[:datasets] << name
+      save
+    end
+    dataset(name)
+  end
+  
+  ##
+  # Unlink dataset identified by +name+ and return MiGA::Dataset.
+  def unlink_dataset(name)
+    d = dataset(name)
+    return nil if d.nil?
+    self.metadata[:datasets].delete(name)
+    save
+    d
+  end
+  
+  ##
+  # Import the dataset +ds+, a MiGA::Dataset, using +method+ which is any method
+  # supported by File#generic_transfer.
+  def import_dataset(ds, method=:hardlink)
+    raise "Impossible to import dataset, it already exists: #{ds.name}." if
+      MiGA::Dataset.exist?(self, ds.name)
+    # Import dataset results
+    ds.each_result do |task, result|
+      # import result files
+      result.each_file do |file|
+        File.generic_transfer("#{result.dir}/#{file}",
+          "#{path}/data/#{MiGA::Dataset.RESULT_DIRS[task]}/#{file}", method)
+      end
+      # import result metadata
+      %w(json start done).each do |suffix|
+        if File.exist? "#{result.dir}/#{ds.name}.#{suffix}"
+          File.generic_transfer("#{result.dir}/#{ds.name}.#{suffix}",
+            "#{path}/data/#{MiGA::Dataset.RESULT_DIRS[task]}/" +
+	                      "#{ds.name}.#{suffix}", method)
+        end
+      end
+    end
+    # Import dataset metadata
+    File.generic_transfer("#{ds.project.path}/metadata/#{ds.name}.json",
+      "#{self.path}/metadata/#{ds.name}.json", method)
+    # Save dataset
+    self.add_dataset(ds.name)
+  end
+
+  ##
+  # Find all datasets with (potential) result files but are yet unregistered.
+  def unregistered_datasets
+    datasets = []
+    MiGA::Dataset.RESULT_DIRS.values.each do |dir|
+      dir_p = "#{path}/data/#{dir}"
+      next unless Dir.exist? dir_p
+      Dir.entries(dir_p).each do |file|
+        next unless
+          file =~ %r{
+            \.(fa(a|sta|stqc?)?|fna|solexaqa|gff[23]?|done|ess)(\.gz)?$
+          }x
+        m = /([^\.]+)/.match(file)
+        datasets << m[1] unless m.nil? or m[1] == "miga-project"
+      end
+    end
+    datasets.uniq - metadata[:datasets]
+  end
+  
+  ##
+  # Are all the datasets in the project preprocessed? Save intermediate results
+  # if +save+ (until the first incomplete dataset is reached).
+  def done_preprocessing?(save=true)
+    dataset_names.each do |dn|
+      ds = dataset(dn)
+      return false if ds.is_ref? and not ds.done_preprocessing?(save)
+    end
+    true
+  end
+
+  ##
+  # Returns a two-dimensional matrix (Array of Array) where the first index
+  # corresponds to the dataset, the second index corresponds to the dataset
+  # task, and the value corresponds to:
+  # - 0: Before execution.
+  # - 1: Done (or not required).
+  # - 2: To do.
+  def profile_datasets_advance
+    advance = []
+    self.each_dataset_profile_advance do |ds_adv|
+      advance << ds_adv
+    end
+    advance
+  end
+
+  ##
+  # Call +blk+ passing the result of MiGA::Dataset#profile_advance for each
+  # registered dataset.
+  def each_dataset_profile_advance(&blk)
+    each_dataset { |ds| blk.call(ds.profile_advance) }
+  end
+  
+end
+

data/lib/miga/project/plugins.rb

@@ -0,0 +1,41 @@
+# @package MiGA
+# @license Artistic-2.0
+
+##
+# Helper module including specific functions handle plugins.
+module MiGA::Project::Plugins
+
+  ##
+  # Installs the plugin in the specified path.
+  def install_plugin(path)
+    abs_path = File.absolute_path(path)
+    raise "Plugin already installed in project: #{abs_path}." unless
+      metadata[:plugins].nil? or not metadata[:plugins].include?(abs_path)
+    raise "Malformed MiGA plugin: #{abs_path}." unless
+      File.exist?(File.expand_path("miga-plugin.json", abs_path))
+    self.metadata[:plugins] ||= []
+    self.metadata[:plugins] << abs_path
+    save
+  end
+
+  ##
+  # Uninstall the plugin in the specified path.
+  def uninstall_plugin(path)
+    abs_path = File.absolute_path(path)
+    raise "Plugin not currently installed: #{abs_path}." if
+      metadata[:plugins].nil? or not metadata[:plugins].include?(abs_path)
+    self.metadata[:plugins].delete(abs_path)
+    save
+  end
+
+  ##
+  # List plugins installed in the project.
+  def plugins ; metadata[:plugins] ||= [] ; end
+
+  ##
+  # Loads the plugins installed in the project.
+  def load_plugins
+    plugins.each { |pl| require File.expand_path("lib-plugin.rb", pl) }
+  end
+
+end

data/lib/miga/{project_result.rb → project/result.rb}

@@ -1,10 +1,75 @@
 # @package MiGA
 # @license Artistic-2.0
 
+require "miga/result"
+require "miga/project/base"
+
 ##
 # Helper module including specific functions to add project results.
-module MiGA::ProjectResult
+module MiGA::Project::Result
+
+  include MiGA::Project::Base
+  
+  ##
+  # Get result identified by Symbol +name+, returns MiGA::Result.
+  def result(name)
+    dir = @@RESULT_DIRS[name.to_sym]
+    return nil if dir.nil?
+    MiGA::Result.load("#{path}/data/#{dir}/miga-project.json")
+  end
 
+  ##
+  # Get all results, an Array of MiGA::Result.
+  def results
+    @@RESULT_DIRS.keys.map{ |k| result(k) }.reject{ |r| r.nil? }
+  end
+  
+  ##
+  # Add the result identified by Symbol +name+, and return MiGA::Result. Save
+  # the result if +save+. The +opts+ hash controls result creation (if
+  # necessary).
+  # Supported values include:
+  # - +force+: A Boolean indicating if the result must be re-indexed. If true,
+  #   it implies save=true.
+  def add_result(name, save=true, opts={})
+    return nil if @@RESULT_DIRS[name].nil?
+    base = "#{path}/data/#{@@RESULT_DIRS[name]}/miga-project"
+    unless opts[:force]
+      r_pre = MiGA::Result.load("#{base}.json")
+      return r_pre if (r_pre.nil? and not save) or not r_pre.nil?
+    end
+    r = result_files_exist?(base, ".done") ?
+        send("add_result_#{name}", base) : nil
+    r.save unless r.nil?
+    r
+  end
+  
+  ##
+  # Get the next distances task, saving intermediate results if +save+. Returns
+  # a Symbol.
+  def next_distances(save=true) ; next_task(@@DISTANCE_TASKS, save) ; end
+  
+  ##
+  # Get the next inclade task, saving intermediate results if +save+. Returns a
+  # Symbol.
+  def next_inclade(save=true) ; next_task(@@INCLADE_TASKS, save) ; end
+
+  ##
+  # Get the next task from +tasks+, saving intermediate results if +save+.
+  # Returns a Symbol.
+  def next_task(tasks=@@DISTANCE_TASKS+@@INCLADE_TASKS, save=true)
+    tasks.find do |t|
+      if metadata["run_#{t}"]==false or
+            (!is_clade? and @@INCLADE_TASKS.include?(t) and
+                  metadata["run_#{t}"]!=true)
+        false
+      else
+        add_result(t, save).nil?
+      end
+    end
+  end
+  
+  
   private
 
     ##

data/lib/miga/version.rb

@@ -10,7 +10,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.3, 1, 3]
+  VERSION = [0.3, 1, 4]
   
   ##
   # Nickname for the current major.minor version.

data/scripts/cds.bash

@@ -11,6 +11,13 @@ cd "$PROJECT/data/06.cds"
 # Initialize
 miga date > "$DATASET.start"
 
+# Gunzip (if necessary)
+if [[ -e "../05.assembly/$DATASET.LargeContigs.fna.gz" \
+      && ! -e "../05.assembly/$DATASET.LargeContigs.fna" ]] ; then
+  gzip -d "../05.assembly/$DATASET.LargeContigs.fna.gz"
+  miga add_result -P "$PROJECT" -D "$DATASET" -r assembly -f
+fi
+
 # Run Prodigal
 TYPE=$(miga list_datasets -P "$PROJECT" -D "$DATASET" \
    --metadata "type" | awk '{print $2}')

data/test/remote_dataset_test.rb

@@ -28,14 +28,16 @@ class RemoteDatasetTest < Test::Unit::TestCase
     assert_raise { MiGA::RemoteDataset.new("ids", :google, :ebi) }
   end
 
-  def test_ebi
+  def test_rest
     hiv2 = "M30502.1"
-    rd = MiGA::RemoteDataset.new(hiv2, :embl, :ebi)
-    assert_equal([hiv2], rd.ids)
-    omit_if(!$remote_tests, "Remote access is error-prone.")
-    tx = rd.get_ncbi_taxonomy
-    assert_equal(MiGA::Taxonomy, tx.class)
-    assert_equal("Lentivirus", tx[:g])
+    {embl: :ebi, nuccore: :ncbi}.each do |db, universe|
+      rd = MiGA::RemoteDataset.new(hiv2, db, universe)
+      assert_equal([hiv2], rd.ids)
+      omit_if(!$remote_tests, "Remote access is error-prone.")
+      tx = rd.get_ncbi_taxonomy
+      assert_equal(MiGA::Taxonomy, tx.class, "Failed on #{universe}:#{db}")
+      assert_equal("Lentivirus", tx[:g], "Failed on #{universe}:#{db}")
+    end
   end
 
   def test_net_ftp

data/utils/distances.rb

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+
+require 'miga'
+
+$project = MiGA::Project.load(ARGV.shift)
+$dataset = $project.dataset(ARGV.shift)
+opts = Hash[ARGV]
+
+exit(0) if dataset.is_multi?
+
+if dataset.is_ref?
+  require_relative 'distances/ref-nomulti.rb'
+else
+  require_relative 'distances/noref-nomulti.rb'
+end
+# TODO run_distances!!!

data/utils/distances/functions.rb

@@ -0,0 +1,58 @@
+
+require 'sqlite3'
+
+$opts = {}
+if ENV["MIGA_AAI_SAVE_RBM"].nil?
+  $opts[:aai_save_rbm] = $project.is_clade? ? "save-rbm" : "no-save-rbm"
+else
+  $opts[:aai_save_rbm] = ENV["MIGA_AAI_SAVE_RBM"]
+end
+$opts[:thr] = ENV["CORES"].nil? ? 2 : ENV["CORES"].to_i
+
+def ani(f1, f2, db, opts={})
+  opts = $opts.merge(opts)
+  v = `ani.rb -1 "#{f1}" -2 "#{f2}" -S "#{db}" --name1 "#{ds_name f1}" --name2 "#{ds_name f2}" \
+        -t "#{opts[:thr]}" -a --no-save-regions --no-save-rbm --lookup-first`
+  v.nil? or v.empty? ? 0 : v.to_f
+end
+
+def make_empty_aai_db(db)
+  SQLite3::Database.new(db) do |conn|
+    conn.execute "create table if not exists aai(" +
+        "seq1 varchar(256), seq2 varchar(256), " +
+        "aai float, sd float, n int omega int" +
+      ")"
+  end unless File.size?(db)
+end
+
+def aai(f1, f2, db, opts={})
+  opts = $opts.merge(opts)
+  v = `aai.rb -1 "#{f1}" -2 "#{f2}" -S "#{db}" --name1 "#{ds_name f1}" --name2 "#{ds_name f2}" \
+        -t "#{opts[:thr]}" -a --lookup-first "--#{opts[:aai_save_rbm]}"`.chomp
+  v.nil? or v.empty? ? 0 : v.to_f
+end
+
+def haai(f1, f2, db, aai_db, opts={})
+  opts = $opts.merge(opts)
+  haai = aai(f1, f2, db, aai_save_rbm: "no-save-rbm")
+  return 0 if haai.nil? or haai == 0 or haai > 90.0
+  aai = 100.0 - Math.exp(2.435076 + 0.4275193*Math.log(100.0-haai))
+  make_empty_aai_db(aai_db)
+  SQLite3::Database.new(db) do |conn|
+    conn.execute "insert into aai values(?, ?, ?, 0, 0, 0)",
+          [ds_name(f1), ds_name(f2), aai]
+  end
+  aai
+end
+
+def haai_or_aai(f1_h, f2_h, db_h, f1, f2, db, opts={})
+  haai=haai(f1_h, f2_h, db_h, db, opts)
+  aai = aai(f1, f2, db, opts) if aai.nil? or aai.zero?
+  aai
+end
+
+def val_from_db(n1, n2, db, metric)
+  SQLite3::Database.new(db) do |conn|
+    return conn.execute("select #{metric} from #{metric} where seq1=? and seq2=?", [n1, n2]).first.first
+  end if File.size? db
+end