miga-base 0.2.5.2 → 0.2.6.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6ad4344d34f36a3ec81feaa2dd60d7e4b0e9f0ee
-  data.tar.gz: fc4cb3a9ff09769199bd24d22aecdd3622f8ea38
+  metadata.gz: 86777e753b2985fdceae0ee27c2d6270baf23e0e
+  data.tar.gz: bc5353979542701b28f35ad0fdc6bc68ee4d0c08
 SHA512:
-  metadata.gz: 02dee40c0c7bc3c6cd7bab93a09317a991a0fbaed66c9b73acf2d605b624fd4035ed76a1e42f4954ed0edc5e6c12002f6a499a898f1c05d855060a04a5c59e92
-  data.tar.gz: 3a255fc1af9aefec2e90ba49ba09640891372cd6bcec4b2c9c91d7ab2e0244ee0be94063e4db7c01a52c229dc7605d7ccf636b1acac0434a179d6da0e0b9fdc9
+  metadata.gz: 9493d3aec9de2f8ba8131ca475bc9d982686168cf64c8c565c64215ecdb34978235419e5134674c9e4d625edf000868f9bd140b7df00cd29aa16f8120d2e27b3
+  data.tar.gz: 9f9125548016a2b29f840c8b8e120527eb3e9dcbbf7cc5a9c464e1bfcbc4e223bca4a3dce3c1e820afe0e7b45cca3307ec53241f5fd3cdb390112714c4353f87

data/actions/create_dataset.rb

@@ -44,6 +44,8 @@ end.parse!
 ##=> Main <=
 opt_require(o)
 opt_require(o, type:"-t") unless o[:update]
+raise "Unrecognized dataset type: #{o[:type]}." if
+  (not o[:update]) and MiGA::Dataset.KNOWN_TYPES[o[:type]].nil?
 
 $stderr.puts "Loading project." unless o[:q]
 p = MiGA::Project.load(o[:project])

data/actions/run_local.rb

@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+
+# @package MiGA
+# @license Artistic-2.0
+
+require "shellwords"
+
+o = {q:true, try_load:false, thr:1}
+opts = OptionParser.new do |opt|
+  opt_banner(opt)
+  opt_object(opt, o, [:project, :dataset_opt, :result])
+  opt.on("-t", "--threads INT",
+    "Threads to use in the local run (by default: #{o[:thr]})."
+    ){ |v| o[:thr] = v.to_i }
+  opt_common(opt, o)
+end.parse!
+
+##=> Main <=
+opts.parse!
+opt_require(o, project:"-P", name:"-r")
+
+$stderr.puts "Loading project." unless o[:q]
+p = MiGA::Project.load(o[:project])
+raise "Impossible to load project: #{o[:project]}" if p.nil?
+
+miga = File.expand_path("../..", __FILE__)
+cmd = ["PROJECT=#{p.path.shellescape}", "RUNTYPE=bash",
+  "MIGA=#{miga.shellescape}", "CORES=#{o[:thr]}"]
+if o[:dataset].nil?
+  type = MiGA::Project
+else
+  d = p.dataset(o[:dataset])
+  raise "Cannot load dataset." if d.nil?
+  cmd << "DATASET=#{d.name.shellescape}"
+  type = MiGA::Dataset
+end
+raise "Unsupported #{type.to_s.gsub(/.*::/,"")} result: #{o[:name]}." if
+  type.RESULT_DIRS[o[:name].to_sym].nil?
+cmd << "#{miga}/scripts/#{o[:name]}.bash".shellescape
+pid = spawn cmd.join(" ")
+Process.wait pid
+
+$stderr.puts "Done." unless o[:q]

data/bin/miga

@@ -28,6 +28,7 @@ $task_desc = {
   result_stats: "Extracts statistics for the given result.",
   list_files: "Lists all registered files from the results of a dataset or a "+
     "project.",
+  run_local: "Executes locally one step analysis producing the given result.",
   # System
   daemon: "Controls the daemon of a MiGA project.",
   date: "Returns the current date in standard MiGA format.",

data/lib/miga/common.rb

@@ -91,7 +91,7 @@ class MiGA::MiGA
             buffer = ""
             tmp.puts ">#{id.gsub(/[^A-Za-z0-9_\|\.]/, "_")}#{df}"
           else
-            buffer += ln.gsub(/[^A-Za-z\.\-]/, "")
+            buffer << ln.gsub(/[^A-Za-z\.\-]/, "")
           end
         end
         tmp.print buffer.wrap_width(80)
@@ -111,7 +111,7 @@ class MiGA::MiGA
   def result_files_exist?(base, ext)
     ext = [ext] unless ext.kind_of? Array
     ext.all? do |f|
-      File.exist?(base + f) or File.exist?(base + f + ".gz")
+      File.exist?(base + f) or File.exist?("#{base}#{f}.gz")
     end
   end
 
@@ -156,7 +156,7 @@ class String
   def miga_name? ; not(self !~ /^[A-Za-z0-9_]+$/) ; end
 
   ##
-  # Replace underscores by spaces.
+  # Replace underscores by spaces or dots (depending on context).
   def unmiga_name ; gsub(/_(str|sp|subsp|pv)__/,"_\\1._").tr("_", " ") ; end
   
   ##

data/lib/miga/daemon.rb

@@ -14,7 +14,7 @@ class MiGA::Daemon < MiGA::MiGA
   # active? Returns DateTime.
   def self.last_alive(project)
     f = File.expand_path("daemon/alive", project.path)
-    return nil unless File.size? f
+    return nil unless File.exist? f
     DateTime.parse(File.read(f))
   end
   
@@ -63,10 +63,9 @@ class MiGA::Daemon < MiGA::MiGA
   def runopts(k, v=nil, force=false)
     k = k.to_sym
     unless v.nil?
-      v = v.to_i if [:latency, :maxjobs, :ppn].include? k
-      raise "Daemon's #{k} cannot be set to zero." if
-        !force and v.is_a? Integer and v==0
-      v = !!v if [:shutdown_when_done].include? k
+      v = [:latency, :maxjobs, :ppn].include?(k) ? v.to_i :
+          [:shutdown_when_done].include?(k) ? !!v : v
+      raise "Daemon's #{k} cannot be set to zero." if !force and v==0
       @runopts[k] = v
     end
     @runopts[k]
@@ -209,23 +208,7 @@ class MiGA::Daemon < MiGA::MiGA
     # Launch as many +jobs_to_run+ as possible
     while jobs_running.size < maxjobs
       break if jobs_to_run.empty?
-      job = @jobs_to_run.shift
-      # Launch job
-      if runopts(:type) == "bash"
-        job[:pid] = spawn job[:cmd]
-        Process.detach job[:pid] unless [nil, "", 0].include? job[:pid]
-      else
-        job[:pid] = `#{job[:cmd]}`.chomp
-      end
-      # Check if registered
-      if [nil, "", 0].include? job[:pid].nil?
-        job[:pid] = nil
-        @jobs_to_run << job
-        say "Unsuccessful #{job[:task_name]}, rescheduling."
-      else
-        @jobs_running << job
-        say "Spawned pid:#{job[:pid]} for #{job[:task_name]}."
-      end
+      launch_job @jobs_to_run.shift
     end
   end
 
@@ -270,4 +253,28 @@ class MiGA::Daemon < MiGA::MiGA
     print "[#{Time.new.inspect}] ", *opts, "\n"
   end
 
+  private
+    
+    def launch_job(job)
+      # Execute job
+      if runopts(:type) == "bash"
+        # Local job
+        job[:pid] = spawn job[:cmd]
+        Process.detach job[:pid] unless [nil, "", 0].include?(job[:pid])
+      else
+        # Schedule cluster job
+        job[:pid] = `#{job[:cmd]}`.chomp
+      end
+      
+      # Check if registered
+      if [nil, "", 0].include?(job[:pid])
+        job[:pid] = nil
+        @jobs_to_run << job
+        say "Unsuccessful #{job[:task_name]}, rescheduling."
+      else
+        @jobs_running << job
+        say "Spawned pid:#{job[:pid]} for #{job[:task_name]}."
+      end
+    end
+
 end

data/lib/miga/dataset.rb

@@ -26,9 +26,6 @@ class MiGA::Dataset < MiGA::MiGA
     ssu: "07.annotation/01.function/02.ssu",
     mytaxa: "07.annotation/02.taxonomy/01.mytaxa",
     mytaxa_scan: "07.annotation/03.qa/02.mytaxa_scan",
-    # Mapping
-    mapping_on_contigs: "08.mapping/01.read-ctg",
-    mapping_on_genes: "08.mapping/02.read-gene",
     # Distances (for single-species datasets)
     distances: "09.distances",
     # General statistics
@@ -59,21 +56,24 @@ class MiGA::Dataset < MiGA::MiGA
   ##
   # Tasks to be excluded from query datasets.
   @@EXCLUDE_NOREF_TASKS = [:mytaxa_scan]
+  @@_EXCLUDE_NOREF_TASKS_H = Hash[@@EXCLUDE_NOREF_TASKS.map{ |i| [i,true] }]
   
   ##
   # Tasks to be executed only in datasets that are not multi-organism. These
   # tasks are ignored for multi-organism datasets or for unknown types.
   @@ONLY_NONMULTI_TASKS = [:mytaxa_scan, :distances]
+  @@_ONLY_NONMULTI_TASKS_H = Hash[@@ONLY_NONMULTI_TASKS.map{ |i| [i,true] }]
 
   ##
   # Tasks to be executed only in datasets that are multi-organism. These
   # tasks are ignored for single-organism datasets or for unknwon types.
   @@ONLY_MULTI_TASKS = [:mytaxa]
+  @@_ONLY_MULTI_TASKS_H = Hash[@@ONLY_MULTI_TASKS.map{ |i| [i,true] }]
 
   ##
   # Does the +project+ already have a dataset with that +name+?
   def self.exist?(project, name)
-    File.exist? project.path + "/metadata/" + name + ".json"
+    File.exist? "#{project.path}/metadata/#{name}.json"
   end
 
   ##
@@ -109,8 +109,6 @@ class MiGA::Dataset < MiGA::MiGA
     metadata[:ref] = is_ref
     @metadata = MiGA::Metadata.new(
       File.expand_path("metadata/#{name}.json", project.path), metadata )
-    warn "Warning: Unrecognized dataset type: #{type}." if
-      !type.nil? and @@KNOWN_TYPES[type].nil?
   end
   
   ##
@@ -165,8 +163,8 @@ class MiGA::Dataset < MiGA::MiGA
   # Get the result MiGA::Result in this dataset identified by the symbol +k+.
   def result(k)
     return nil if @@RESULT_DIRS[k.to_sym].nil?
-    MiGA::Result.load(project.path + "/data/" + @@RESULT_DIRS[k.to_sym] +
-      "/" + name + ".json")
+    MiGA::Result.load(
+      "#{project.path}/data/#{@@RESULT_DIRS[k.to_sym]}/#{name}.json" )
   end
   
   ##
@@ -184,16 +182,18 @@ class MiGA::Dataset < MiGA::MiGA
   ##
   # Look for the result with symbol key +result_type+ and register it in the
   # dataset. If +save+ is false, it doesn't register the result, but it still
-  # returns a result if the expected files are complete. Returns MiGA::Result
-  # or nil.
-  def add_result(result_type, save=true)
-    return nil if @@RESULT_DIRS[result_type].nil?
-    base = File.expand_path("data/#{@@RESULT_DIRS[result_type]}/#{name}",
-              project.path)
+  # returns a result if the expected files are complete. The +opts+ array
+  # controls result creation (if necessary). Supported values include:
+  # - +is_clean+: A Boolean indicating if the input files are clean.
+  # Returns MiGA::Result or nil.
+  def add_result(result_type, save=true, opts={})
+    dir = @@RESULT_DIRS[result_type]
+    return nil if dir.nil?
+    base = File.expand_path("data/#{dir}/#{name}", project.path)
     r_pre = MiGA::Result.load("#{base}.json")
     return r_pre if (r_pre.nil? and not save) or not r_pre.nil?
-    return nil unless result_files_exist?(base, ".done")
-    r = self.send("add_result_#{result_type}", base)
+    r = File.exist?("#{base}.done") ?
+        self.send("add_result_#{result_type}", base, opts) : nil
     r.save unless r.nil?
     r
   end
@@ -232,11 +232,12 @@ class MiGA::Dataset < MiGA::MiGA
   # Should I ignore +task+ for this dataset?
   def ignore_task?(task)
     return !metadata["run_#{task}"] unless metadata["run_#{task}"].nil?
-    ( (@@EXCLUDE_NOREF_TASKS.include?(task) and not is_ref?) or
-      (@@ONLY_MULTI_TASKS.include?(task) and not is_multi?) or
-      (@@ONLY_NONMULTI_TASKS.include?(task) and not is_nonmulti?))
+    pattern = [true, false]
+    ( [@@_EXCLUDE_NOREF_TASKS_H[task], is_ref?     ]==pattern or
+      [@@_ONLY_MULTI_TASKS_H[task],    is_multi?   ]==pattern or
+      [@@_ONLY_NONMULTI_TASKS_H[task], is_nonmulti?]==pattern )
   end
-  
+
   ##
   # Are all the dataset-specific tasks done? Passes +save+ to #add_result.
   def done_preprocessing?(save=false)

data/lib/miga/dataset_result.rb

@@ -27,10 +27,10 @@ module MiGA::DatasetResult
   private
 
     ##
-    # Add result type +:raw_reads+ at +base+.
-    def add_result_raw_reads(base)
+    # Add result type +:raw_reads+ at +base+ (no +opts+ supported).
+    def add_result_raw_reads(base, opts)
       return nil unless result_files_exist?(base, ".1.fastq")
-      r = MiGA::Result.new(base + ".json")
+      r = MiGA::Result.new("#{base}.json")
       r = add_files_to_ds_result(r, name,
         ( result_files_exist?(base, ".2.fastq") ?
           {:pair1=>".1.fastq", :pair2=>".2.fastq"} :
@@ -38,24 +38,24 @@ module MiGA::DatasetResult
     end
 
     ##
-    # Add result type +:trimmed_reads+ at +base+.
-    def add_result_trimmed_reads(base)
+    # Add result type +:trimmed_reads+ at +base+ (no +opts+ supported).
+    def add_result_trimmed_reads(base, opts)
       return nil unless result_files_exist?(base, ".1.clipped.fastq")
-      r = MiGA::Result.new base + ".json"
+      r = MiGA::Result.new("#{base}.json")
       r = add_files_to_ds_result(r, name,
         {:pair1=>".1.clipped.fastq", :pair2=>".2.clipped.fastq"}) if
         result_files_exist?(base, ".2.clipped.fastq")
-      r.add_file(:single, name + ".1.clipped.single.fastq")
-      r.add_file(:trimming_sumary, name + ".1.fastq.trimmed.summary.txt")
+      r.add_file(:single, "#{name}.1.clipped.single.fastq")
+      r.add_file(:trimming_sumary, "#{name}.1.fastq.trimmed.summary.txt")
       add_result(:raw_reads) #-> Post gunzip
       r
     end
 
     ##
-    # Add result type +:read_quality+ at +base+.
-    def add_result_read_quality(base)
+    # Add result type +:read_quality+ at +base+ (no +opts+ supported).
+    def add_result_read_quality(base, opts)
       return nil unless result_files_exist?(base, %w[.solexaqa .fastqc])
-      r = MiGA::Result.new(base + ".json")
+      r = MiGA::Result.new("#{base}.json")
       r = add_files_to_ds_result(r, name,
         {:solexaqa=>".solexaqa", :fastqc=>".fastqc"})
       add_result(:trimmed_reads) #-> Post cleaning
@@ -63,13 +63,13 @@ module MiGA::DatasetResult
     end
 
     ##
-    # Add result type +:trimmed_fasta+ at +base+.
-    def add_result_trimmed_fasta(base)
+    # Add result type +:trimmed_fasta+ at +base+ (no +opts+ supported).
+    def add_result_trimmed_fasta(base, opts)
       return nil unless
         result_files_exist?(base, ".CoupledReads.fa") or
         result_files_exist?(base, ".SingleReads.fa") or
         result_files_exist?(base, %w[.1.fasta .2.fasta])
-      r = MiGA::Result.new base + ".json"
+      r = MiGA::Result.new("#{base}.json")
       r = add_files_to_ds_result(r, name, {:coupled=>".CoupledReads.fa",
         :single=>".SingleReads.fa", :pair1=>".1.fasta", :pair2=>".2.fasta"})
       add_result(:raw_reads) #-> Post gzip
@@ -77,12 +77,15 @@ module MiGA::DatasetResult
     end
 
     ##
-    # Add result type +:assembly+ at +base+.
-    def add_result_assembly(base)
+    # Add result type +:assembly+ at +base+. Hash +opts+ supports
+    # +is_clean: Boolean+.
+    def add_result_assembly(base, opts)
       return nil unless result_files_exist?(base, ".LargeContigs.fna")
-      r = MiGA::Result.new(base + ".json")
+      r = MiGA::Result.new("#{base}.json")
       r = add_files_to_ds_result(r, name, {:largecontigs=>".LargeContigs.fna",
         :allcontigs=>".AllContigs.fna", :assembly_data=>""})
+      opts[:is_clean] ||= false
+      r.clean! if opts[:is_clean]
       unless r.clean?
         MiGA::MiGA.clean_fasta_file(r.file_path :largecontigs)
         r.clean!
@@ -92,12 +95,14 @@ module MiGA::DatasetResult
     end
 
     ##
-    # Add result type +:cds+ at +base+.
-    def add_result_cds(base)
+    # Add result type +:cds+ at +base+. Hash +opts+ supports +is_clean: Boolean+
+    def add_result_cds(base, opts)
       return nil unless result_files_exist?(base, %w[.faa .fna])
-      r = MiGA::Result.new(base + ".json")
+      r = MiGA::Result.new("#{base}.json")
       r = add_files_to_ds_result(r, name, {:proteins=>".faa", :genes=>".fna",
         :gff2=>".gff2", :gff3=>".gff3", :tab=>".tab"})
+      opts[:is_clean] ||= false
+      r.clean! if opts[:is_clean]
       unless r.clean?
         MiGA::MiGA.clean_fasta_file(r.file_path :proteins)
         MiGA::MiGA.clean_fasta_file(r.file_path :genes)
@@ -107,22 +112,24 @@ module MiGA::DatasetResult
     end
 
     ##
-    # Add result type +:essential_genes+ at +base+.
-    def add_result_essential_genes(base)
+    # Add result type +:essential_genes+ at +base+ (no +opts+ supported).
+    def add_result_essential_genes(base, opts)
       return nil unless result_files_exist?(base, %w[.ess.faa .ess .ess/log])
-      r = MiGA::Result.new(base + ".json")
+      r = MiGA::Result.new("#{base}.json")
       r = add_files_to_ds_result(r, name, {:ess_genes=>".ess.faa",
         :collection=>".ess", :report=>".ess/log"})
     end
 
     ##
-    # Add result type +:ssu+ at +base+.
-    def add_result_ssu(base)
-      return MiGA::Result.new(base + ".json") if result(:assembly).nil?
+    # Add result type +:ssu+ at +base+. Hash +opts+ supports +is_clean: Boolean+
+    def add_result_ssu(base, opts)
+      return MiGA::Result.new("#{base}.json") if result(:assembly).nil?
       return nil unless result_files_exist?(base, ".ssu.fa")
-      r = MiGA::Result.new(base + ".json")
+      r = MiGA::Result.new("#{base}.json")
       r = add_files_to_ds_result(r, name, {:longest_ssu_gene=>".ssu.fa",
         :gff=>".ssu.gff", :all_ssu_genes=>".ssu.all.fa"})
+      opts[:is_clean] ||= false
+      r.clean! if opts[:is_clean]
       unless r.clean?
         MiGA::MiGA.clean_fasta_file(r.file_path :longest_ssu_gene)
         r.clean!
@@ -131,37 +138,37 @@ module MiGA::DatasetResult
     end
 
     ##
-    # Add result type +:mytaxa+ at +base+.
-    def add_result_mytaxa(base)
+    # Add result type +:mytaxa+ at +base+ (no +opts+ supported).
+    def add_result_mytaxa(base, opts)
       if is_multi?
         return nil unless result_files_exist?(base, ".mytaxa")
-        r = MiGA::Result.new(base + ".json")
+        r = MiGA::Result.new("#{base}.json")
         add_files_to_ds_result(r, name, {:mytaxa=>".mytaxa", :blast=>".blast",
           :mytaxain=>".mytaxain"})
       else
-        MiGA::Result.new(base + ".json")
+        MiGA::Result.new("#{base}.json")
       end
     end
 
     ##
-    # Add result type +:mytaxa_scan+ at +base+.
-    def add_result_mytaxa_scan(base)
+    # Add result type +:mytaxa_scan+ at +base+ (no +opts+ supported).
+    def add_result_mytaxa_scan(base, opts)
       if is_nonmulti?
         return nil unless
           result_files_exist?(base, %w[.pdf .wintax .mytaxa .reg])
-        r = MiGA::Result.new(base + ".json")
+        r = MiGA::Result.new("#{base}.json")
         add_files_to_ds_result(r, name, {:mytaxa=>".mytaxa", :wintax=>".wintax",
           :blast=>".blast", :mytaxain=>".mytaxain", :report=>".pdf",
           :regions=>".reg", :gene_ids=>".wintax.genes",
           :region_ids=>".wintax.regions"})
       else
-        MiGA::Result.new base + ".json"
+        MiGA::Result.new("#{base}.json")
       end
     end
 
     ##
-    # Add result type +:distances+ at +base+.
-    def add_result_distances(base)
+    # Add result type +:distances+ at +base+ (no +opts+ supported).
+    def add_result_distances(base, opts)
       if is_nonmulti?
         if is_ref?
           add_result_distances_ref(base)
@@ -174,15 +181,15 @@ module MiGA::DatasetResult
     end
 
     ##
-    # Add result type +:stats+ at +base+.
-    def add_result_stats(base)
-      MiGA::Result.new "#{base}.json"
+    # Add result type +:stats+ at +base+ (no +opts+ supported).
+    def add_result_stats(base, opts)
+      MiGA::Result.new("#{base}.json")
     end
     
     ##
     # Add result type +:distances+ for _multi_ datasets at +base+.
     def add_result_distances_multi(base)
-      MiGA::Result.new "#{base}.json"
+      MiGA::Result.new("#{base}.json")
     end
 
     ##
@@ -191,7 +198,7 @@ module MiGA::DatasetResult
       pref = File.dirname(base)
       return nil unless
         File.exist?("#{pref}/01.haai/#{name}.db")
-      r = MiGA::Result.new(base + ".json")
+      r = MiGA::Result.new("#{base}.json")
       r.add_files({:haai_db=>"01.haai/#{name}.db",
         :aai_db=>"02.aai/#{name}.db", :ani_db=>"03.ani/#{name}.db"})
       r
@@ -203,7 +210,7 @@ module MiGA::DatasetResult
       return nil unless
         result_files_exist?(base, %w[.aai-medoids.tsv .aai.db]) or
         result_files_exist?(base, %w[.ani-medoids.tsv .ani.db])
-      r = MiGA::Result.new(base + ".json")
+      r = MiGA::Result.new("#{base}.json")
       r = add_files_to_ds_result(r, name, {
         :aai_medoids=>".aai-medoids.tsv",
         :haai_db=>".haai.db", :aai_db=>".aai.db",

data/lib/miga/metadata.rb

@@ -9,7 +9,7 @@ class MiGA::Metadata < MiGA::MiGA
 
   ##
   # Does the metadata described in +path+ already exist?
-  def self.exist?(path) File.size? path end
+  def self.exist?(path) File.exist? path end
 
   ##
   # Load the metadata described in +path+ and return MiGA::Metadata if it
@@ -24,34 +24,39 @@ class MiGA::Metadata < MiGA::MiGA
   ##
   # Path to the JSON file describing the metadata.
   attr_reader :path
-  
-  ##
-  # Parsed data as a Hash.
-  attr_reader :data
 
   ##
   # Initiate a MiGA::Metadata object with description in +path+. It will create
   # it if it doesn't exist.
   def initialize(path, defaults={})
+    @data = nil
     @path = File.absolute_path(path)
-    @data = {}
-    defaults.each_pair{ |k,v| self[k]=v }
-    self.create unless File.size? self.path
-    self.load
+    unless File.exist? path
+      @data = {}
+      defaults.each_pair{ |k,v| self[k]=v }
+      create
+    end
+  end
+  
+  ##
+  # Parsed data as a Hash.
+  def data
+    self.load if @data.nil?
+    @data
   end
 
   ##
   # Reset :created field and save the current data.
   def create
-    @data[:created] = Time.now.to_s
-    self.save
+    self[:created] = Time.now.to_s
+    save
   end
 
   ##
   # Save the metadata into #path.
   def save
     MiGA.DEBUG "Metadata.save #{path}"
-    @data[:updated] = Time.now.to_s
+    self[:updated] = Time.now.to_s
     json = JSON.pretty_generate(data)
     sleeper = 0.0
     while File.exist?(lock_file)
@@ -59,12 +64,12 @@ class MiGA::Metadata < MiGA::MiGA
       sleep(sleeper.to_i)
     end
     FileUtils.touch lock_file
-    ofh = File.open(path + ".tmp", "w")
+    ofh = File.open("#{path}.tmp", "w")
     ofh.puts json
     ofh.close
     raise "Lock-racing detected for #{path}." unless
-      File.exist?(path + ".tmp") and File.exist?(lock_file)
-    File.rename(path + ".tmp", path)
+      File.exist?("#{path}.tmp") and File.exist?(lock_file)
+    File.rename("#{path}.tmp", path)
     File.unlink(lock_file)
   end
 
@@ -86,14 +91,14 @@ class MiGA::Metadata < MiGA::MiGA
   ##
   # Delete file at #path.
   def remove!
-    MiGA.DEBUG "Metadata.remove! #{self.path}"
-    File.unlink(self.path)
+    MiGA.DEBUG "Metadata.remove! #{path}"
+    File.unlink(path)
     nil
   end
 
   ##
   # Lock file for the metadata.
-  def lock_file ; path + ".lock" ; end
+  def lock_file ; "#{path}.lock" ; end
 
   ##
   # Return the value of +k+ in #data.
@@ -102,6 +107,7 @@ class MiGA::Metadata < MiGA::MiGA
   ##
   # Set the value of +k+ to +v+.
   def []=(k,v)
+    self.load if @data.nil?
     k = k.to_sym
     # Protect the special field :name
     v=v.miga_name if k==:name

data/lib/miga/project.rb

@@ -49,11 +49,11 @@ class MiGA::Project < MiGA::MiGA
     clade_finding: "10.clades/01.find",
     # Clade analysis
     subclades: "10.clades/02.ani",
-    ogs: "10.clades/03.ogs"
+    ogs: "10.clades/03.ogs",
     #ess_phylogeny: "10.clades/04.phylogeny/01.essential",
     #core_phylogeny: "10.clades/04.phylogeny/02.core",
     #clade_metadata: "10.clades/05.metadata"
-    #project_stats: "90.stats"
+    project_stats: "90.stats"
   }
 
   ##
@@ -85,7 +85,7 @@ class MiGA::Project < MiGA::MiGA
   ##
   # Does the project at +path+ exist?
   def self.exist?(path)
-    Dir.exist?(path) and File.exist?(path + "/miga.project.json")
+    Dir.exist?(path) and File.exist?("#{path}/miga.project.json")
   end
 
   ##
@@ -130,7 +130,7 @@ class MiGA::Project < MiGA::MiGA
     dirs.each{ |d| Dir.mkdir(d) unless Dir.exist? d }
     @metadata = MiGA::Metadata.new(self.path + "/miga.project.json",
       {datasets: [], name: File.basename(path)})
-    FileUtils.cp(ENV["MIGA_HOME"] + "/.miga_daemon.json",
+    FileUtils.cp("#{ENV["MIGA_HOME"]}/.miga_daemon.json",
       "#{path}/daemon/daemon.json") unless
         File.exist? "#{path}/daemon/daemon.json"
     self.load
@@ -252,9 +252,9 @@ class MiGA::Project < MiGA::MiGA
   ##
   # Get result identified by Symbol +name+, returns MiGA::Result.
   def result(name)
-    return nil if @@RESULT_DIRS[name.to_sym].nil?
-    MiGA::Result.load "#{path}/data/" + @@RESULT_DIRS[name.to_sym] + 
-      "/miga-project.json"
+    dir = @@RESULT_DIRS[name.to_sym]
+    return nil if dir.nil?
+    MiGA::Result.load("#{path}/data/#{dir}/miga-project.json")
   end
   
   ##
@@ -269,7 +269,7 @@ class MiGA::Project < MiGA::MiGA
   def add_result(name, save=true)
     return nil if @@RESULT_DIRS[name].nil?
     base = "#{path}/data/#{@@RESULT_DIRS[name]}/miga-project"
-    return MiGA::Result.load(base + ".json") unless save
+    return MiGA::Result.load("#{base}.json") unless save
     return nil unless result_files_exist?(base, ".done")
     r = send("add_result_#{name}", base)
     r.save unless r.nil?

data/lib/miga/project_result.rb

@@ -11,7 +11,7 @@ module MiGA::ProjectResult
     # Internal alias for all add_result_*_distances.
     def add_result_distances(base)
       return nil unless result_files_exist?(base, %w[.Rdata .log .txt])
-      r = MiGA::Result.new(base + ".json")
+      r = MiGA::Result.new("#{base}.json")
       r.add_file(:rdata, "miga-project.Rdata")
       r.add_file(:matrix, "miga-project.txt")
       r.add_file(:log, "miga-project.log")
@@ -40,7 +40,7 @@ module MiGA::ProjectResult
     end
 
     def add_result_iter_clades(base)
-      r = MiGA::Result.new(base + ".json")
+      r = MiGA::Result.new("#{base}.json")
       r.add_file(:report,	"miga-project.pdf")
       r.add_file(:class_table,	"miga-project.class.tsv")
       r.add_file(:class_tree,	"miga-project.class.nwk")
@@ -51,13 +51,22 @@ module MiGA::ProjectResult
 
     def add_result_ogs(base)
       return nil unless result_files_exist?(base, %w[.ogs .stats])
-      r = MiGA::Result.new(base + ".json")
+      r = MiGA::Result.new("#{base}.json")
       r.add_file(:ogs, "miga-project.ogs")
       r.add_file(:stats, "miga-project.stats")
       r.add_file(:rbm, "miga-project.rbm")
       r
     end
 
+    def add_result_project_stats(base)
+      return nil unless
+        result_files_exist?(base, %w[.taxonomy.json .metadata.db])
+      r = MiGA::Result.new("#{base}.json")
+      r.add_file(:taxonomy_index, "miga-project.taxonomy.json")
+      r.add_file(:metadata_index, "miga-project.metadata.db")
+      r
+    end
+
     alias add_result_haai_distances add_result_distances
     alias add_result_aai_distances add_result_distances
     alias add_result_ani_distances add_result_distances

data/lib/miga/remote_dataset.rb

@@ -9,6 +9,7 @@ require "open-uri"
 class MiGA::RemoteDataset < MiGA::MiGA
   # Class-level
 
+  @@_EUTILS = "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/"
   ##
   # Structure of the different database Universes or containers. The structure
   # is a Hash with universe names as keys as Symbol and values being a Hash with
@@ -37,16 +38,14 @@ class MiGA::RemoteDataset < MiGA::MiGA
     },
     ncbi:{
       dbs: { nuccore:{stage: :assembly, format: :fasta} },
-      url: "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/" +
-        "efetch.fcgi?db=%1$s&id=%2$s&rettype=%3$s&retmode=text",
+      url: "#{@@_EUTILS}efetch.fcgi?db=%1$s&id=%2$s&rettype=%3$s&retmode=text",
       method: :rest
     },
     ncbi_map:{
       dbs: { assembly:{map_to: :nuccore, format: :text} },
-      url: "https://eutils.ncbi.nlm.nih.gov/entrez/eutils/" +
         # FIXME ncbi_map is intended to do internal NCBI mapping between
         # databases.
-        "elink.fcgi?dbfrom=%1$s&id=%2$s&db=%3$s - - - - -",
+      url: "#{@@_EUTILS}elink.fcgi?dbfrom=%1$s&id=%2$s&db=%3$s - - - - -",
       method: :rest,
       map_to_universe: :ncbi
     }
@@ -127,8 +126,8 @@ class MiGA::RemoteDataset < MiGA::MiGA
     metadata = get_metadata(metadata)
     case @@UNIVERSE[universe][:dbs][db][:stage]
     when :assembly
-      base = project.path + "/data/" + MiGA::Dataset.RESULT_DIRS[:assembly] +
-        "/" + name
+      dir = MiGA::Dataset.RESULT_DIRS[:assembly]
+      base = "#{project.path}/data/#{dir}/#{name}"
       File.open("#{base}.start", "w") { |ofh| ofh.puts Time.now.to_s }
       if @@UNIVERSE[universe][:dbs][db][:format] == :fasta_gz
         download("#{base}.LargeContigs.fna.gz")
@@ -144,9 +143,12 @@ class MiGA::RemoteDataset < MiGA::MiGA
     end
     dataset = MiGA::Dataset.new(project, name, is_ref, metadata)
     project.add_dataset(dataset.name)
-    result = dataset.add_result @@UNIVERSE[universe][:dbs][db][:stage]
+    result = dataset.add_result(@@UNIVERSE[universe][:dbs][db][:stage],
+      true, is_clean:true)
     raise "Empty dataset created: seed result was not added due to "+
       "incomplete files." if result.nil?
+    result.clean!
+    result.save
     dataset
   end
 

data/lib/miga/result.rb

@@ -9,9 +9,7 @@ class MiGA::Result < MiGA::MiGA
   
   ##
   # Check if the result described by the JSON in +path+ already exists.
-  def self.exist?(path)
-    !!(File.size? path)
-  end
+  def self.exist?(path) File.exist? path end
 
   ##
   # Load the result described by the JSON in +path+. Returns MiGA::Result if it
@@ -82,8 +80,8 @@ class MiGA::Result < MiGA::MiGA
     k = k.to_sym
     @data[:files] ||= {}
     @data[:files][k] = file if File.exist? File.expand_path(file, dir)
-    @data[:files][k] = file + ".gz" if
-      File.exist? File.expand_path(file + ".gz", dir)
+    @data[:files][k] = "#{file}.gz" if
+      File.exist? File.expand_path("#{file}.gz", dir)
   end
 
   ##

data/lib/miga/tax_index.rb

@@ -115,13 +115,13 @@ class MiGA::TaxIndexTaxon < MiGA::MiGA
   ##
   # Get the number of datasets in the taxon (including children).
   def datasets_count
-    datasets.size + children.map{ |it| it.datasets_count }.reduce(0, :+)
+    children.map{ |it| it.datasets_count }.reduce(datasets.size, :+)
   end
 
   ##
   # Get all the datasets in the taxon (including children).
   def all_datasets
-    datasets + children.map{ |it| it.datasets }.reduce([], :+)
+    children.map{ |it| it.datasets }.reduce(datasets, :+)
   end
 
   ##
@@ -142,11 +142,11 @@ class MiGA::TaxIndexTaxon < MiGA::MiGA
   # Tabular String of the taxon.
   def to_tab(unknown, indent=0)
     o = ""
-    o = (" " * indent) + tax_str + ": " + datasets_count.to_s + "\n" if
+    o = "#{" " * indent}#{tax_str}: #{datasets_count}\n" if
       unknown or not datasets.empty? or not name.nil?
     indent += 2
-    datasets.each{ |ds| o += (" " * indent) + "# " + ds.name + "\n" }
-    children.each{ |it| o += it.to_tab(unknown, indent) }
+    datasets.each{ |ds| o << "#{" " * indent}# #{ds.name}\n" }
+    children.each{ |it| o << it.to_tab(unknown, indent) }
     o
   end
 

data/lib/miga/taxonomy.rb

@@ -10,6 +10,7 @@ class MiGA::Taxonomy < MiGA::MiGA
   # Cannonical ranks.
   def self.KNOWN_RANKS() @@KNOWN_RANKS ; end
   @@KNOWN_RANKS = %w{ns d k p c o f g s ssp str ds}.map{|r| r.to_sym}
+  @@_KNOWN_RANKS_H = Hash[ @@KNOWN_RANKS.map{ |i| [i,true] } ]
 
   ##
   # Long names of the cannonical ranks.
@@ -42,11 +43,12 @@ class MiGA::Taxonomy < MiGA::MiGA
   ##
   # Returns cannonical rank (Symbol) for the +rank+ String.
   def self.normalize_rank(rank)
+    return rank.to_sym if @@_KNOWN_RANKS_H[rank.to_sym]
     rank = rank.to_s.downcase
     return nil if rank=="no rank"
     rank = @@RANK_SYNONYMS[rank] unless @@RANK_SYNONYMS[rank].nil?
     rank = rank.to_sym
-    return nil unless @@KNOWN_RANKS.include? rank
+    return nil unless @@_KNOWN_RANKS_H[rank]
     rank
   end
 
@@ -84,16 +86,16 @@ class MiGA::Taxonomy < MiGA::MiGA
   # Add +value+ to the hierarchy, that can be an Array, a String, or a Hash, as
   # described in #initialize.
   def <<(value)
-    if value.is_a? Array
-      value.each{ |v| self << v }
-    elsif value.is_a? String
-      (rank, name) = value.split(/:/)
-      self << { rank => name }
-    elsif value.is_a? Hash
+    if value.is_a? Hash
       value.each_pair do |rank_i, name_i|
         next if name_i.nil? or name_i == ""
         @ranks[ Taxonomy.normalize_rank rank_i ] = name_i.tr("_"," ")
       end
+    elsif value.is_a? Array
+      value.each{ |v| self << v }
+    elsif value.is_a? String
+      (rank, name) = value.split(/:/)
+      self << { rank => name }
     else
       raise "Unsupported class: #{value.class.name}."
     end

data/lib/miga/version.rb

@@ -10,7 +10,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.2, 5, 2]
+  VERSION = [0.2, 6, 0]
   
   ##
   # Nickname for the current major.minor version.
@@ -18,7 +18,7 @@ module MiGA
   
   ##
   # Date of the current gem release.
-  VERSION_DATE = Date.new(2017, 4, 3)
+  VERSION_DATE = Date.new(2017, 4, 14)
   
   ##
   # Reference of MiGA.
@@ -42,8 +42,7 @@ class MiGA::MiGA
   ##
   # Complete version with nickname and date as string.
   def self.LONG_VERSION
-    "MiGA " + VERSION.join(".") + " - " + VERSION_NAME + " - " +
-      VERSION_DATE.to_s
+    "MiGA #{VERSION.join(".")} - #{VERSION_NAME} - #{VERSION_DATE}"
   end
 
   ##

data/scripts/essential_genes.bash

@@ -1,7 +1,7 @@
 #!/bin/bash
 # Available variables: $PROJECT, $RUNTYPE, $MIGA, $CORES
 set -e
-SCRIPT="essential"
+SCRIPT="essential_genes"
 echo "MiGA: $MIGA"
 echo "Project: $PROJECT"
 source "$MIGA/scripts/miga.bash" || exit 1

data/scripts/project_stats.bash

@@ -0,0 +1,24 @@
+#!/bin/bash
+# Available variables: $PROJECT, $RUNTYPE, $MIGA, $CORES
+set -e
+SCRIPT="project_stats"
+echo "MiGA: $MIGA"
+echo "Project: $PROJECT"
+source "$MIGA/scripts/miga.bash" || exit 1
+DIR="$PROJECT/data/90.stats"
+[[ -d "$DIR" ]] || mkdir -p "$DIR"
+cd "$DIR"
+
+# Initialize
+miga date > "miga-project.start"
+
+# Index taxonomy
+miga index_taxonomy -P "$PROJECT" -i "miga-project.taxonomy.json" --ref
+
+# Index metadata
+ruby -I "$MIGA/lib"
+  "$MIGA/utils/index_metadata.rb" "$PROJECT" "miga-project.metadata.db"
+
+# Finalize
+miga date > "miga-project.done"
+miga add_result -P "$PROJECT" -r "$SCRIPT"

data/test/dataset_test.rb

@@ -92,13 +92,24 @@ class DatasetTest < Test::Unit::TestCase
     assert_equal(:trimmed_reads, d2.first_preprocessing(true))
     assert_equal(:read_quality, d2.next_preprocessing(true))
     assert(! d2.done_preprocessing?(true))
+    # Ref and undeclared multi
     assert(d2.ignore_task?(:mytaxa))
+    assert(d2.ignore_task?(:mytaxa_scan))
     assert(d2.ignore_task?(:distances))
+    # Ref and multi
     d2.metadata[:type] = :metagenome
     assert(! d2.ignore_task?(:mytaxa))
+    assert(d2.ignore_task?(:mytaxa_scan))
     assert(d2.ignore_task?(:distances))
+    # Ref and nonmulti
     d2.metadata[:type] = :genome
     assert(d2.ignore_task?(:mytaxa))
+    assert(! d2.ignore_task?(:mytaxa_scan))
+    assert(! d2.ignore_task?(:distances))
+    # Qry and nonmulti
+    d2.metadata[:ref] = false
+    assert(d2.ignore_task?(:mytaxa))
+    assert(d2.ignore_task?(:mytaxa_scan))
     assert(! d2.ignore_task?(:distances))
   end
 

data/utils/index_metadata.rb

@@ -0,0 +1,28 @@
+#!/usr/bin/env ruby
+
+require "miga"
+require "sqlite3"
+
+p = MiGA::Project.load(ARGV[0])
+raise "Impossible to load project: #{ARGV[0]}." if p.nil?
+
+File.unlink(ARGV[1]) if File.exist? ARGV[1]
+db = SQLite3::Database.new(ARGV[1])
+db.execute "create table metadata(" +
+  "`name` varchar(256), `field` varchar(256), `value` text)"
+
+def searchable(db, k, v)
+  db.execute "insert into metadata values(?,?,?)",
+    k.to_s, " #{v.to_s.downcase.gsub(/[^A-Za-z0-9\-]+/, " ")} "
+end
+
+p.each_dataset do |name, d|
+  next unless d.is_ref?
+  searchable(db, :name, d.name)
+  d.metadata.each do |k, v|
+    next if [:created, :updated].include? k
+    v = v.sorted_ranks.map{ |r| r[1] }.join(" ") if k==:tax
+    searchable(db, k, v)
+  end
+end
+

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 0.2.5.2
+  version: 0.2.6.0
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-04-03 00:00:00.000000000 Z
+date: 2017-04-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rest-client
@@ -149,6 +149,7 @@ files:
 - scripts/mytaxa.bash
 - scripts/mytaxa_scan.bash
 - scripts/ogs.bash
+- scripts/project_stats.bash
 - scripts/read_quality.bash
 - scripts/ssu.bash
 - scripts/stats.bash
@@ -156,6 +157,7 @@ files:
 - scripts/trimmed_fasta.bash
 - scripts/trimmed_reads.bash
 - utils/adapters.fa
+- utils/index_metadata.rb
 - utils/mytaxa_scan.R
 - utils/mytaxa_scan.rb
 - utils/plot-taxdist.R
@@ -179,6 +181,7 @@ files:
 - actions/plugins.rb
 - actions/project_info.rb
 - actions/result_stats.rb
+- actions/run_local.rb
 - actions/tax_distributions.rb
 - actions/unlink_dataset.rb
 - Gemfile