miga-base 0.2.0.6

data/lib/miga/gui.rb

@@ -0,0 +1,289 @@
+#
+# @package MiGA
+# @author Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @license artistic license 2.0
+# @update Jul-11-2015
+#
+
+require 'miga/project'
+require 'shoes'
+require 'shoes/swt'
+
+Shoes::Swt.initialize_backend
+
+module MiGA
+   class MiGA
+      @@STATUS = "Initializing MiGA..."
+      def self.STATUS=(status) @@STATUS = status ; end
+      def self.STATUS() @@STATUS; end
+      def self.RESET_STATUS() @@STATUS="MiGA is ready to go!" ; end
+   end
+   class GUI < Shoes
+      url '/', :index
+      url '/project', :project
+      url '/datasets', :datasets
+      url '/dataset-(.*)', :dataset
+      url '/report', :report
+      $miga_path = File.expand_path(File.dirname(__FILE__) + "/../../")
+
+      def self.init (&block)
+	 Shoes.app title: "MiGA | Microbial Genomes Atlas", width: 750, height: 400, &block
+      end
+      
+      # =====================[ View : Windows ]
+      # Main window
+      def index
+	 status_bar
+	 stack(margin:40) do
+	    title "Welcome to MiGA!", align:"center"
+	    para ""
+	    flow{ menu_bar [:open_project, :new_project, :help] }
+	    para ""
+	    stack do
+	       image $miga_path + "/gui/img/MiGA.png", width:150, height:50
+	       para ""
+	       para "If you use MiGA in your research, please consider citing:"
+	       para MiGA.CITATION
+	       para ""
+	    end
+	 end
+	 MiGA.RESET_STATUS
+	 keypress do |key|
+	    if [:control_o, "o"].include? key
+	       MiGA.STATUS = "Calling open_project..."
+	       open_project
+	    elsif [:control_n, "n"].include? key
+	       MiGA.STATUS = "Calling new_project..."
+	       new_project
+	    end
+	 end
+      end
+
+      # Project window
+      def project
+	 stack(margin:40) do
+	    title $project.name.unmiga_name, align:"center"
+	    caption $project.path, align:"center"
+	    para ""
+	    stack{ menu_bar [:list_datasets, :new_dataset, :progress_report, :help] }
+	    para ""
+	    stack do
+	       para strong("Datasets"), ": ", $project.metadata[:datasets].size
+	       $project.metadata.each { |k,v| para(strong(k), ": ", v) unless k==:datasets }
+	    end
+	    para ""
+	 end
+	 MiGA.RESET_STATUS
+	 keypress do |key|
+	    if [:control_r, "r"].include? key
+	       MiGA.STATUS = "Calling progress_report..."
+	       progress_report
+	    elsif [:control_d, "d"].include? key
+	       MiGA.STATUS = "Calling list_datasets..."
+	       list_datasets
+	    end
+	 end
+      end
+
+      # Datasets list window
+      def datasets
+	 stack(margin:40) do
+	    title $project.name.unmiga_name, align:"center"
+	    caption $project.path, align:"center"
+	    para ""
+	    stack do
+	       para "Displaying #{$project.metadata[:datasets].size} datasets:"
+	       para ""
+	       $project.metadata[:datasets].each do |name|
+		  para link(name, :click=>"/dataset-#{name}")
+	       end
+	    end
+	    para ""
+	    MiGA.RESET_STATUS
+	 end
+      end
+
+      # Dataset details window
+      def dataset(name)
+	 stack(margin:40) do
+	    ds = $project.dataset(name)
+	    title ds.name.unmiga_name, align:"center"
+	    caption "A dataset in ", strong(link($project.name.unmiga_name, :click=>"/datasets")), align:"center"
+	    para ""
+	    stack{ ds.metadata.each { |k,v| para strong(k), ": ", v } }
+	    para ""
+	    flow do
+	       w = 40+30*Dataset.PREPROCESSING_TASKS.size
+	       stack(width:w) do
+		  subtitle "Advance"
+		  done = self.graphic_advance(ds)
+		  para sprintf("%.1f%% Complete", done*100)
+	       end
+	       stack(width:-w) do
+		  subtitle "Task"
+		  $task_name_field = stack { para "" }
+		  animate do
+		     $task_name_field.clear{ para $task }
+		  end
+	       end
+	    end
+	    para ""
+	    MiGA.RESET_STATUS
+	 end
+      end
+
+      # Project report window
+      def report
+	 stack(margin:40) do
+	    title $project.name.unmiga_name, align:"center"
+	    $done = 0.0
+	    $me = self
+	    flow do
+	       para ""
+	       w = 40+30*Dataset.PREPROCESSING_TASKS.size
+	       stack(width:w) do
+		  para ""
+		  subtitle "Dataset tasks advance:"
+		  caption link("toggle"){ $adv_logo.toggle }
+		  para ""
+		  $adv_logo = stack do
+		     $project.each_dataset do |ds|
+			$done += $me.graphic_advance(ds, 1)
+		     end
+		     motion do |x,y|
+			unless $task.nil?
+			   $task_ds_box.clear do
+			      subtitle "Task"
+			      para $task
+			      subtitle "Dataset"
+			      para $dataset
+			   end
+			   $task_ds_box.show
+			   $task_ds_box.move(w,y-150)
+			end
+		     end
+		     click do
+			GUI.init{ visit "/dataset-#{$dataset}" } unless $dataset.nil?
+		     end
+		     leave do
+			$task = nil
+			$dataset = nil
+			$task_ds_box.hide
+		     end
+		  end
+		  $done /= $project.metadata[:datasets].size
+		  para sprintf("%.1f%% Complete", $done*100)
+	       end
+	       $task_ds_box = stack(width:-w)
+	    end
+	    if $done==1.0
+	       para ""
+	       stack do
+		  subtitle "Project-wide tasks:"
+		  Project.DISTANCE_TASKS.each { |t| para strong(t), ": ", ($project.add_result(t).nil? ? "Pending" : "Done") }
+		  if $project.metadata[:type]==:clade
+		     Project.INCLADE_TASKS.each { |t| para strong(t), ": ", ($project.add_result(t).nil? ? "Pending" : "Done") }
+		  end
+	       end
+	    end
+	    para ""
+	    MiGA.RESET_STATUS
+	 end
+      end
+      
+      # =====================[ View : Elements ]
+      # Menu bar
+      def menu_bar actions
+	 flow do
+	    b = {
+	       open_project:["Open project", "iconmonstr-archive-5-icon-256"],
+	       new_project:["New project","iconmonstr-plus-5-icon-256"],
+	       list_datasets:["List datasets", "iconmonstr-note-10-icon-256"],
+	       new_dataset:["New dataset", "iconmonstr-note-25-icon-256"],
+	       progress_report:["Progress report", "iconmonstr-bar-chart-2-icon-256"],
+	       help:["Help", "iconmonstr-help-3-icon-256"]
+	    }
+	    actions.each do |k|
+	       v = b[k]
+	       flow(margin:0, width:200) do
+		  image $miga_path + "/gui/img/#{v[1]}.png", width:40, height:40, margin:2
+		  button v[0], top:5 do
+		     MiGA.STATUS = "Calling #{k}..."
+		     eval k.to_s
+		  end
+	       end
+	    end
+	 end
+      end # menu_bar
+      
+      # Status bar
+      def status_bar
+	 stack(bottom:0) do
+	    flow bottom:0, height:20, margin:0 do
+	       background "#CCC"
+	       stack(width:50)
+	       $status_cont = stack(width:-300, height:1.0)
+	       every do |i|
+		  $status_cont.clear { inscription MiGA.STATUS, margin:5 }
+	       end
+	       stack(width:250, height:1.0) do
+		  inscription MiGA.LONG_VERSION, align:"right", margin:5
+	       end
+	    end
+	    image $miga_path + "/gui/img/MiGA-sq.png", left:0, bottom:0, width:30, height:32
+	 end
+      end # status_bar
+
+      def graphic_advance(ds, h=10)
+	 ds_adv = ds.profile_advance
+	 flow(width:30*Dataset.PREPROCESSING_TASKS.size) do
+	    nostroke
+	    i = 0
+	    col = ["#CCC", rgb(119,130,61), rgb(160,41,50)]
+	    ds_adv.each do |j|
+	       stack(width:28,margin:0,top:0,left:i*30,height:h) do
+		  background col[j]
+		  t = Dataset.PREPROCESSING_TASKS[i]
+		  hover do
+		     $task = t
+		     $dataset = ds.name.unmiga_name
+		  end
+	       end
+	       i += 1
+	    end
+	    nofill
+	 end
+	 return 0.0 if ds_adv.count{|i| i>0} <= 1
+	 (ds_adv.count{|i| i==1}.to_f - 1.0)/(ds_adv.count{|i| i>0}.to_f - 1.0)
+      end # graphic_advance
+
+      # =====================[ Controller : Projects ]
+      def open_project
+	 GUI.init do
+	    folder = ask_open_folder
+	    if folder.nil? or not Project.exist? folder
+	       alert "Cannot find a MiGA project at #{folder}!" unless folder.nil?
+	       return
+	    else
+	       $project = Project.new folder
+	       visit '/project'
+	    end
+	 end
+      end # open_project
+      def new_project
+	 GUI.init do
+	    folder = ask_save_folder
+	    if folder.nil? or Project.exist? folder
+	       alert "Cannot overwrite existent MiGA project at #{folder}!" unless folder.nil?
+	       return
+	    else
+	       $project = Project.new folder
+	       visit "/project"
+	    end
+	 end
+      end # new_project
+      def list_datasets ; GUI.init{ visit "/datasets" } ; end # list_datasets
+      def progress_report ; GUI.init{ visit "/report" } ; end # progress_report
+   end
+end
+

data/lib/miga/metadata.rb

@@ -0,0 +1,74 @@
+#
+# @package MiGA
+# @author Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @license artistic license 2.0
+# @update Jul-06-2015
+#
+
+module MiGA
+   class Metadata
+      # Class
+      def self.exist?(path) File.size? path end
+      def self.load path
+	 return nil unless Metadata.exist? path
+	 Metadata.new path
+      end
+      # Instance
+      attr_reader :path, :data
+      def initialize(path, defaults={})
+	 @path = File.absolute_path(path)
+	 @data = {}
+	 defaults.each_pair{ |k,v| self[k]=v }
+	 self.create unless File.size? self.path
+	 self.load
+      end
+      def create
+         @data[:created] = Time.now.to_s
+	 self.save
+      end
+      def save
+	 MiGA.DEBUG "Metadata.save #{self.path}"
+         self.data[:updated] = Time.now.to_s
+	 json = JSON.pretty_generate(self.data)
+	 sleeper = 0.0
+	 while File.exist? self.path + ".lock"
+	    sleeper += 0.1 if sleeper <= 10.0
+	    sleep(sleeper.to_i)
+	 end
+	 FileUtils.touch self.path + ".lock"
+	 ofh = File.open(self.path + ".tmp", "w")
+	 ofh.puts json
+	 ofh.close
+	 raise "Lock-racing detected for #{self.path}." unless File.exist? self.path + ".tmp" and File.exist? self.path + ".lock"
+	 File.rename self.path + ".tmp", self.path
+	 File.unlink self.path + ".lock"
+      end
+      def load
+	 sleeper = 0.0
+	 while File.exist? self.path + ".lock"
+	    sleeper += 0.1 if sleeper <= 10.0
+	    sleep(sleeper.to_i)
+	 end
+	 # :symbolize_names does not play nicely with :create_additions
+	 tmp = JSON.parse File.read(self.path), {:symbolize_names=>false, :create_additions=>true}
+	 @data = {}
+	 tmp.each_pair{ |k,v| self[k] = v }
+      end
+      def remove!
+	 MiGA.DEBUG "Metadata.remove! #{self.path}"
+	 File.unlink self.path
+      end
+      def [](k) self.data[k.to_sym] end
+      def []=(k,v)
+	 k = k.to_sym
+	 # Protect the special field :name
+	 v=v.miga_name if k==:name
+	 # Symbolize the special field :type
+	 v=v.to_sym if k==:type
+	 # Register and return
+	 self.data[k]=v
+      end
+      def each(&blk) self.data.each{ |k,v| blk.call(k,v) } ; end
+   end
+end
+

data/lib/miga/project.rb

@@ -0,0 +1,268 @@
+#
+# @package MiGA
+# @author  Luis M. Rodriguez-R <lmrodriguezr at gmail dot com>
+# @license artistic license 2.0
+# @update  Jan-18-2016
+#
+
+require "miga/dataset"
+
+module MiGA
+   class Project
+      # Class
+      @@FOLDERS = %w(data metadata daemon)
+      @@DATA_FOLDERS = %w(
+	 01.raw_reads 02.trimmed_reads 03.read_quality 04.trimmed_fasta
+	 05.assembly 06.cds
+	 07.annotation 07.annotation/01.function 07.annotation/02.taxonomy
+	 07.annotation/01.function/01.essential
+	 07.annotation/01.function/02.ssu
+	 07.annotation/02.taxonomy/01.mytaxa
+	 07.annotation/03.qa 07.annotation/03.qa/01.checkm
+	 07.annotation/03.qa/02.mytaxa_scan
+	 08.mapping 08.mapping/01.read-ctg 08.mapping/02.read-gene
+	 09.distances 09.distances/01.haai 09.distances/02.aai
+	 09.distances/03.ani 09.distances/04.ssu
+	 10.clades 10.clades/01.find 10.clades/02.ani 10.clades/03.ogs
+	 10.clades/04.phylogeny 10.clades/04.phylogeny/01.essential
+	 10.clades/04.phylogeny/02.core 10.clades/05.metadata)
+      @@RESULT_DIRS = {
+	 # Distances
+	 haai_distances: "09.distances/01.haai",
+	 aai_distances: "09.distances/02.aai",
+	 ani_distances: "09.distances/03.ani",
+	 #ssu_distances: "09.distances/04.ssu",
+	 # Clade identification
+	 clade_finding: "10.clades/01.find",
+	 # Clade analysis
+	 subclades: "10.clades/02.ani",
+	 ogs: "10.clades/03.ogs",
+	 ess_phylogeny: "10.clades/04.phylogeny/01.essential",
+	 core_phylogeny: "10.clades/04.phylogeny/02.core",
+	 clade_metadata: "10.clades/05.metadata"
+      }
+      @@KNOWN_TYPES = {
+	 mixed: {description: "Mixed collection of genomes, metagenomes, " +
+	    "and viromes.",
+	    single: true, multi: true},
+	 genomes: {description: "Collection of genomes.",
+	    single: true, multi: false},
+	 clade: {description: "Collection of closely-related genomes " +
+	    "(ANI <= 90%).",
+	    single: true, multi: false},
+	 metagenomes: {description: "Collection of metagenomes and/or " +
+	    "viromes.",
+	    single: false, multi: true}
+      }
+      @@DISTANCE_TASKS = [:haai_distances, :aai_distances, :ani_distances,
+	 :clade_finding]
+      @@INCLADE_TASKS = [:subclades, :ogs, :ess_phylogeny, :core_phylogeny,
+	 :clade_metadata]
+      def self.DISTANCE_TASKS ; @@DISTANCE_TASKS ; end
+      def self.INCLADE_TASKS ; @@INCLADE_TASKS ; end
+      def self.RESULT_DIRS ; @@RESULT_DIRS ; end
+      def self.KNOWN_TYPES ; @@KNOWN_TYPES ; end
+      def self.exist?(path)
+	 Dir.exist?(path) and File.exist?(path + "/miga.project.json")
+      end
+      def self.load(path)
+	 return nil unless Project.exist? path
+	 Project.new path
+      end
+      # Instance
+      attr_reader :path, :metadata
+      def initialize(path, update=false)
+         raise "Impossible to create project in uninitialized MiGA." unless
+	    File.exist? "#{ENV["HOME"]}/.miga_rc" and
+	    File.exist? "#{ENV["HOME"]}/.miga_daemon.json"
+	 @path = File.absolute_path(path)
+	 self.create if update or not Project.exist? self.path
+	 self.load if self.metadata.nil?
+      end
+      def create
+	 Dir.mkdir self.path unless Dir.exist? self.path
+	 @@FOLDERS.each do |dir|
+	    Dir.mkdir self.path + "/" + dir unless
+	       Dir.exist? self.path + "/" + dir
+	 end
+	 @@DATA_FOLDERS.each do |dir|
+	    Dir.mkdir self.path + "/data/" + dir unless
+	       Dir.exist? self.path + "/data/" + dir
+	 end
+	 @metadata = Metadata.new(self.path + "/miga.project.json",
+	    {datasets: [], name: File.basename(self.path)})
+	 FileUtils.cp(ENV["HOME"] + "/.miga_daemon.json",
+	    self.path + "/daemon/daemon.json") unless
+	    File.exist? self.path + "/daemon/daemon.json"
+	 self.load
+      end
+      def save
+	 self.metadata.save
+	 self.load
+      end
+      def load
+	 @metadata = Metadata.load self.path + "/miga.project.json"
+	 raise "Couldn't find project metadata at #{self.path}" if
+	    self.metadata.nil?
+      end
+      def name ; self.metadata[:name] ; end
+      def datasets
+	 self.metadata[:datasets].map{ |name| self.dataset name }
+      end
+      def dataset(name)
+	 name = name.miga_name
+	 @datasets = {} if @datasets.nil?
+	 @datasets[name] = Dataset.new(self, name) if @datasets[name].nil? 
+	 @datasets[name]
+      end
+      def each_dataset(&blk)
+	 self.metadata[:datasets].each{ |name| blk.call(self.dataset name) }
+      end
+      def add_dataset(name)
+	 self.metadata[:datasets] << name unless
+	    self.metadata[:datasets].include? name
+	 self.save
+	 self.dataset(name)
+      end
+      def unlink_dataset(name)
+	 d = self.dataset name
+	 return nil if d.nil?
+	 self.metadata[:datasets].delete(name)
+	 self.save
+	 d
+      end
+      def import_dataset(ds, method=:hardlink)
+	 raise "Impossible to import dataset, it already exists: #{ds.name}." if
+	    Dataset.exist?(self, ds.name)
+	 # Import dataset results
+	 ds.each_result do |task, result|
+	    # import result files
+	    result.each_file do |file|
+	       File.generic_transfer("#{result.dir}/#{file}",
+		  "#{self.path}/data/#{Dataset.RESULT_DIRS[task]}/#{file}",
+		  method)
+	    end
+	    # import result metadata
+	    %w(json start done).each do |suffix|
+	       if File.exist? "#{result.dir}/#{ds.name}.#{suffix}"
+		  File.generic_transfer("#{result.dir}/#{ds.name}.#{suffix}",
+		     "#{self.path}/data/#{Dataset.RESULT_DIRS[task]}/" +
+			"#{ds.name}.#{suffix}",
+		     method)
+	       end
+	    end
+	 end
+	 # Import dataset metadata
+	 File.generic_transfer("#{ds.project.path}/metadata/#{ds.name}.json",
+	    "#{self.path}/metadata/#{ds.name}.json", method)
+	 # Save dataset
+	 self.add_dataset ds.name 
+      end
+      def result(name)
+	 return nil if @@RESULT_DIRS[name.to_sym].nil?
+	 Result.load self.path + "/data/" + @@RESULT_DIRS[name.to_sym] + 
+	    "/miga-project.json"
+      end
+      def results
+	 @@RESULT_DIRS.keys.map{ |k| self.result k }.reject{ |r| r.nil? }
+      end
+      def add_result result_type
+	 return nil if @@RESULT_DIRS[result_type].nil?
+	 base = self.path + "/data/" + @@RESULT_DIRS[result_type] +
+	    "/miga-project"
+	 return nil unless File.exist? base + ".done"
+	 r = nil
+	 case result_type
+	 when :haai_distances, :aai_distances, :ani_distances, :ssu_distances
+	    return nil unless
+	       File.exist? base + ".Rdata" and
+	       File.exist? base + ".log" and
+	       (File.exist?(base + ".txt") or File.exist?(base + ".txt.gz"))
+	    r = Result.new base + ".json"
+	    r.add_file :rdata, "miga-project.Rdata"
+	    r.add_file :matrix, "miga-project.txt"
+	    r.add_file :log, "miga-project.log"
+	    r.add_file :hist, "miga-project.hist"
+	    r.data[:gz] = File.exist?(base + ".txt.gz")
+	 when :clade_finding
+	    return nil unless File.exist? base + ".proposed-clades"
+	    r = Result.new base + ".json"
+	    r.add_file :proposal, "miga-project.proposed-clades"
+	    r.add_file :rbm_aai90, "genome-genome.aai90.rbm"
+	    r.add_file :clades_aai90, "miga-project.ani-clades"
+	    r.add_file :rbm_ani95, "genome-genome.ani95.rbm"
+	    r.add_file :clades_ani95, "miga-project.ani95-clades"
+	 when :subclades
+	    return nil unless
+	       File.exist?(base+".pdf") and
+	       File.exist?(base+".1.classif") and
+	       File.exist?(base+".1.medoids") and
+	       File.exist?(base+".class.tsv") and
+	       File.exist?(base+".class.nwk")
+	    r = Result.new base + ".json"
+	    r.add_file :report, "miga-project.pdf"
+	    (1..6).each do |i|
+	       %w{classif medoids}.each do |m|
+		  r.add_file "#{m}_#{i}".to_sym, "miga-project.#{i}.#{m}"
+	       end
+	    end
+	    r.add_file :class_table, "miga-project.class.tsv"
+	    r.add_file :class_tree,  "miga-project.class.nwk"
+	    r.add_file :ani_tree,    "miga-project.ani.nwk"
+	 when :ogs
+	    return nil unless
+	       File.exist?(base+".ogs") and
+	       File.exist?(base+".stats")
+	    r = Result.new base + ".json"
+	    r.add_file :ogs, "miga-project.ogs"
+	    r.add_file :stats, "miga-project.stats"
+	    r.add_file :rbm, "miga-project.rbm"
+	 end
+	 r.save
+	 r
+      end
+      def next_distances
+	 @@DISTANCE_TASKS.find{ |t| self.add_result(t).nil? }
+      end
+      def next_inclade
+         return nil unless self.metadata[:type]==:clade
+	 @@INCLADE_TASKS.find{ |t| self.add_result(t).nil? }
+      end
+      def unregistered_datasets
+	 datasets = []
+	 Dataset.RESULT_DIRS.each do |res, dir|
+	    Dir.entries(self.path + "/data/" + dir).each do |file|
+	       next unless
+		  file =~ %r{
+		     \.(fa(a|sta|stqc?)?|fna|solexaqa|gff[23]?|done|ess)(\.gz)?$
+		     }x
+	       m = /([^\.]+)/.match(file)
+	       datasets << m[1] unless m.nil? or m[1] == "miga-project"
+	    end
+	 end
+	 datasets.uniq - self.metadata[:datasets]
+      end
+      def done_preprocessing?
+	 self.datasets.map{|ds| (not ds.is_ref?) or ds.done_preprocessing?}.all?
+      end
+      ## Generates a two-dimensional matrix (array of arrays) where the first
+      ## index corresponds to the dataset, the second index corresponds to the
+      ## dataset task, and the value corresponds to:
+      ##   0: Before execution.
+      ##   1: Done (or not required).
+      ##   2: To do.
+      def profile_datasets_advance
+	 advance = []
+	 self.each_dataset_profile_advance do |ds_adv|
+	    advance << ds_adv
+	 end
+	 advance
+      end
+      def each_dataset_profile_advance(&blk)
+         self.each_dataset do |ds|
+	    blk.call(ds.profile_advance)
+	 end
+      end
+   end
+end
+