miga-base 0.3.9.1 → 0.3.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 89d27413bd8dc321a75fc3dc434a191c1eba336bfab3beaaa1e815852048e30a
-  data.tar.gz: 59eeab21ff1ef81bf76147c1b98c3a49e31535831fe19e8ec2757b59a666dd4e
+  metadata.gz: 27d4c5ad304eb34c537ac2e625501b21682421567c555eb7152453d00b65a49e
+  data.tar.gz: bacf38077a05bec6c25dcbdd630b7a4f18f140ed4588877ac78254f3d9fc381d
 SHA512:
-  metadata.gz: 2e4d975743a230c951578605e93a1b4f3061e8e3d1c91cf147db5c297bfa2427de344a72f118c50d0a3b3e9b2e67137baed9cc14157e86aa3d98738bb09d334f
-  data.tar.gz: d9761542220413d20b162bceeada84cf414d9d17b10764d049b3a76c2f9a3d757389b24c0ac290e02801246c114496e4729c45da5e5b863c22fde009c2eeca66
+  metadata.gz: 9b4d8fe08eaa514a76d3c6b62d0788aa44eb2f95e4cb56eb256671033a6f2ee29482c8ffc5186c614d4f587a4af1a6d85dd1b87a980fd2edc11ebb94330b4e07
+  data.tar.gz: a02db245387ff5185541ad98fb7556d018320eb69763cafbb37a96e1fb2fe8918b9701f13de6513a97ecd5902104414b47d380a62abf7528bb0dab5d35dc61cd

data/actions/daemon.rb

@@ -3,53 +3,53 @@
 # @package MiGA
 # @license Artistic-2.0
 
-require "miga/daemon"
+require 'miga/daemon'
 
-task = ARGV.shift unless ["-h","--help"].include? ARGV.first
-ARGV << "-h" if ARGV.empty?
-o = {q:true, daemon_opts:[]}
+task = ARGV.shift unless %w[-h --help].include? ARGV.first
+ARGV << '-h' if ARGV.empty?
+o = {q: true, daemon_opts: []}
 OptionParser.new do |opt|
   opt_banner(opt)
-  opt.separator "task:"
-  { start:  "Start an instance of the application.",
-    stop:    "Start an instance of the application.",
-    restart: "Stop all instances and restart them afterwards.",
-    reload:  "Send a SIGHUP to all instances of the application.",
-    run:     "Start the application and stay on top.",
-    zap:     "Set the application to a stopped state.",
-    status:  "Show status (PID) of application instances."
-  }.each{ |k,v| opt.separator sprintf "    %*s%s", -33, k, v }
-  opt.separator ""
-  opt.separator "MiGA options:"
+  opt.separator 'task:'
+  { start:   'Start an instance of the application.',
+    stop:    'Start an instance of the application.',
+    restart: 'Stop all instances and restart them afterwards.',
+    reload:  'Send a SIGHUP to all instances of the application.',
+    run:     'Start the application and stay on top.',
+    zap:     'Set the application to a stopped state.',
+    status:  'Show status (PID) of application instances.'
+  }.each{ |k,v| opt.separator sprintf '    %*s%s', -33, k, v }
+  opt.separator ''
+  opt.separator 'MiGA options:'
   opt_object(opt, o, [:project])
-  opt.on("--shutdown-when-done",
-    "If passed, the daemon will exit when all processing is done.",
-    "Otherwise (default), it will stay idle awaiting for new data."
+  opt.on('--shutdown-when-done',
+    'If passed, the daemon will exit when all processing is done.',
+    'Otherwise (default), it will stay idle awaiting for new data.'
     ){ |v| o[:shutdown_when_done] = v }
-  opt.on("--latency INT",
-    "Number of seconds the daemon will be sleeping."
+  opt.on('--latency INT',
+    'Number of seconds the daemon will be sleeping.'
     ){ |v| o[:latency]=v.to_i }
-  opt.on("--max-jobs INT",
-    "Maximum number of jobs to use simultaneously."){ |v| o[:maxjobs]=v.to_i }
-  opt.on("--ppn INT",
-    "Maximum number of cores to use in a single job."){ |v| o[:ppn]=v.to_i }
+  opt.on('--max-jobs INT',
+    'Maximum number of jobs to use simultaneously.'){ |v| o[:maxjobs]=v.to_i }
+  opt.on('--ppn INT',
+    'Maximum number of cores to use in a single job.'){ |v| o[:ppn]=v.to_i }
   opt_common(opt, o)
-  opt.separator "Daemon options:"
-  opt.on("-t", "--ontop",
-    "Stay on top (does not daemonize)."){ o[:daemon_opts] << '-t' }
-  opt.on("-f", "--force", "Force operation."){ o[:daemon_opts] << '-f' }
-  opt.on("-n", "--no_wait",
-    "Do not wait for processes to stop."){ o[:daemon_opts] << '-n' }
-  opt.on("--shush", "Silence the daemon."){ o[:daemon_opts] << '--shush' }
+  opt.separator 'Daemon options:'
+  opt.on('-t', '--ontop',
+    'Stay on top (does not daemonize).'){ o[:daemon_opts] << '-t' }
+  opt.on('-f', '--force', 'Force operation.'){ o[:daemon_opts] << '-f' }
+  opt.on('-n', '--no_wait',
+    'Do not wait for processes to stop.'){ o[:daemon_opts] << '-n' }
+  opt.on('--shush', 'Silence the daemon.'){ o[:daemon_opts] << '--shush' }
 end.parse!
 
 ##=> Main <=
-opt_require(o, project:"-P")
+opt_require(o, project: '-P')
 
-raise "Project doesn't exist, aborting." unless MiGA::Project.exist? o[:project]
-p = MiGA::Project.new(o[:project])
+p = MiGA::Project.load(o[:project]) or raise 'Project doesn\'t exist, aborting.'
 d = MiGA::Daemon.new(p)
 [:latency, :maxjobs, :ppn, :shutdown_when_done].each do |k|
   d.runopts(k, o[k]) unless o[k].nil?
 end
 d.daemon(task, o[:daemon_opts])
+

data/actions/get.rb

@@ -5,7 +5,8 @@
 
 require 'miga/remote_dataset'
 
-o = {q: true, query: false, universe: :ebi, db: :embl, get_md: false}
+o = {q: true, query: false, universe: :ebi, db: :embl,
+  get_md: false, only_md: false}
 OptionParser.new do |opt|
   opt_banner(opt)
   opt_object(opt, o, [:project, :dataset, :dataset_type])
@@ -39,6 +40,9 @@ OptionParser.new do |opt|
   opt.on('--get-metadata',
     'Only download and update metadata for existing datasets'
     ){ |v| o[:get_md] = v }
+  opt.on('--only-metadata',
+    'Create datasets without input data but retrieve all metadata.'
+    ){ |v| o[:only_md] = v }
   opt.on('--api-key STRING',
     'API key for the given universe.'){ |v| o[:api_key] = v }
   opt_common(opt, o)
@@ -91,6 +95,7 @@ glob.each do |o_i|
     $stderr.puts 'Creating dataset.' unless o_i[:q]
     dummy_d = MiGA::Dataset.new(p, o_i[:dataset])
     md = add_metadata(o_i, dummy_d).metadata.data
+    md[:metadata_only] = true if o[:only_md]
     dummy_d.remove!
     rd.save_to(p, o_i[:dataset], !o_i[:query], md)
     p.add_dataset(o_i[:dataset])

data/actions/ncbi_get.rb

@@ -6,11 +6,11 @@
 require 'miga/remote_dataset'
 require 'csv'
 
-o = {q:true, query:false, unlink:false,
+o = {q: true, query: false, unlink: false,
   reference: false, legacy_name: false,
   complete: false, chromosome: false,
   scaffold: false, contig: false, add_version: true, dry: false,
-  get_md: false}
+  get_md: false, only_md: false, save_every: 1}
 OptionParser.new do |opt|
   opt_banner(opt)
   opt_object(opt, o, [:project])
@@ -43,6 +43,13 @@ OptionParser.new do |opt|
   opt.on('--get-metadata',
     'Only download and update metadata for existing datasets'
     ){ |v| o[:get_md] = v }
+  opt.on('--only-metadata',
+    'Create datasets without input data but retrieve all metadata.'
+    ){ |v| o[:only_md] = v }
+  opt.on('--save-every INT',
+    'Save project every this many downloaded datasets.',
+    'If zero, it saves the project only once upon completion.',
+    'By default: 1.'){ |v| o[:save_every] = v.to_i }
   opt.on('-q', '--query',
     'Register the datasets as queries, not reference datasets.'
     ){ |v| o[:query]=v }
@@ -61,6 +68,7 @@ opt_require(o, taxon: '-T') unless o[:reference]
 unless %w[reference complete chromosome scaffold contig].any?{ |i| o[i.to_sym] }
   raise 'No action requested. Pick at least one type of genome.'
 end
+o[:save_every] = 1 if o[:dry]
 
 ##=> Main <=
 $stderr.puts "Loading project." unless o[:q]
@@ -100,6 +108,7 @@ doc = MiGA::RemoteDataset.download_url(url)
 CSV.parse(doc, headers: true).each do |r|
   asm = r['assembly']
   next if asm.nil? or asm.empty? or asm == '-'
+  next unless r['ftp_path_genbank']
 
   # Get replicons
   rep = r['replicons'].nil? ? nil : r['replicons'].
@@ -141,14 +150,16 @@ end
 
 # Download entries
 $stderr.puts "Downloading #{ds.size} " +
-  (ds.size == 1 ? "entry" : "entries") unless o[:q]
-ds.each do |name,body|
+  (ds.size == 1 ? 'entry' : 'entries') unless o[:q]
+p.do_not_save = true if o[:save_every] <= 1
+ds.each do |name, body|
   d << name
   puts name
   next if p.dataset(name).nil? == o[:get_md]
   downloaded += 1
   next if o[:dry]
   $stderr.puts '  Locating remote dataset.' unless o[:q]
+  body[:md][:metadata_only] = true if o[:only_md]
   rd = MiGA::RemoteDataset.new(body[:ids], body[:db], body[:universe])
   if o[:get_md]
     $stderr.puts '  Updating dataset.' unless o[:q]
@@ -158,8 +169,12 @@ ds.each do |name,body|
     rd.save_to(p, name, !o[:query], body[:md])
     p.add_dataset(name)
   end
+  p.save! if o[:save_every] > 1 and (downloaded % o[:save_every]) == 0
 end
 
+p.do_not_save = false
+p.save! if o[:save_every] != 1
+
 # Finalize
 $stderr.puts "Datasets listed: #{d.size}" unless o[:q]
 $stderr.puts "Datasets #{o[:dry] ? 'to download' : 'downloaded'}: " +

data/actions/rm.rb

@@ -3,26 +3,27 @@
 # @package MiGA
 # @license Artistic-2.0
 
-o = {q:true, remove:false}
+o = {q: true, remove: false}
 OptionParser.new do |opt|
   opt_banner(opt)
   opt_object(opt, o)
-  opt.on("-r", "--remove", "Also remove all associated files.",
-    "By default, only unlinks from metadata."){ o[:remove]=true }
+  opt.on('-r', '--remove', 'Also remove all associated files.',
+    'By default, only unlinks from metadata.'){ o[:remove] = true }
   opt_common(opt, o)
 end.parse!
 
 ##=> Main <=
 opt_require(o)
 
-$stderr.puts "Loading project." unless o[:q]
+$stderr.puts 'Loading project.' unless o[:q]
 p = MiGA::Project.load(o[:project])
 raise "Impossible to load project: #{o[:project]}" if p.nil?
 
-$stderr.puts "Unlinking dataset." unless o[:q]
-raise "Dataset doesn't exist, aborting." unless
+$stderr.puts 'Unlinking dataset.' unless o[:q]
+raise 'Dataset doesn\'t exist, aborting.' unless
   MiGA::Dataset.exist?(p, o[:dataset])
 d = p.unlink_dataset(o[:dataset])
 d.remove! if o[:remove]
 
-$stderr.puts "Done." unless o[:q]
+$stderr.puts 'Done.' unless o[:q]
+

data/lib/miga/common.rb

@@ -1,8 +1,8 @@
 # @package MiGA
 # @license Artistic-2.0
 
-require 'json'
 require 'miga/version'
+require 'miga/json'
 require 'miga/common/base'
 require 'miga/common/path'
 require 'miga/common/format'

data/lib/miga/daemon/base.rb

@@ -11,7 +11,7 @@ module MiGA::Daemon::Base
   # Set/get #options, where +k+ is the Symbol of the option and +v+ is the value
   # (or nil to use as getter). Skips consistency tests if +force+. Returns new
   # value.
-  def runopts(k, v=nil, force=false)
+  def runopts(k, v = nil, force = false)
     k = k.to_sym
     unless v.nil?
       if [:latency, :maxjobs, :ppn].include?(k)
@@ -19,16 +19,9 @@ module MiGA::Daemon::Base
       elsif [:shutdown_when_done].include?(k)
         v = !!v
       end
-      raise "Daemon's #{k} cannot be set to zero." if !force and v==0
+      raise "Daemon's #{k} cannot be set to zero." if !force and v == 0
       @runopts[k] = v
     end
-    if k==:kill and v.nil?
-      case @runopts[:type].to_s
-      when 'bash' then return "kill -9 '%s'"
-      when 'qsub' then return "qdel '%s'"
-      else             return "canceljob '%s'"
-      end
-    end
     @runopts[k]
   end
 
@@ -51,19 +44,19 @@ module MiGA::Daemon::Base
 
   ##
   # Initializes the daemon with +opts+.
-  def start(opts=[]) daemon('start', opts); end
+  def start(opts = []) daemon('start', opts); end
 
   ##
   # Stops the daemon with +opts+.
-  def stop(opts=[]) daemon('stop', opts); end
+  def stop(opts = []) daemon('stop', opts); end
 
   ##
   # Restarts the daemon with +opts+.
-  def restart(opts=[]) daemon('restart', opts); end
+  def restart(opts = []) daemon('restart', opts); end
 
   ##
   # Returns the status of the daemon with +opts+.
-  def status(opts=[]) daemon('status', opts); end
+  def status(opts = []) daemon('status', opts); end
 
 end
 

data/lib/miga/daemon.rb

@@ -39,9 +39,9 @@ class MiGA::Daemon < MiGA::MiGA
   def initialize(project)
     $_MIGA_DAEMON_LAIR << self
     @project = project
-    @runopts = JSON.parse(
-          File.read(File.expand_path('daemon/daemon.json', project.path)),
-          symbolize_names: true)
+    @runopts = MiGA::Json.parse(
+      File.expand_path('daemon/daemon.json', project.path),
+      default: File.expand_path('.miga_daemon.json', ENV['MIGA_HOME']))
     @jobs_to_run = []
     @jobs_running = []
     @loop_i = -1
@@ -84,10 +84,9 @@ class MiGA::Daemon < MiGA::MiGA
   ##
   # Report status in a JSON file.
   def report_status
-    f = File.open(File.expand_path('daemon/status.json', project.path), 'w')
-    f.print JSON.pretty_generate(
-      jobs_running: @jobs_running, jobs_to_run: @jobs_to_run)
-    f.close
+    MiGA::Json.generate(
+      {jobs_running: @jobs_running, jobs_to_run: @jobs_to_run},
+      File.expand_path('daemon/status.json', project.path))
   end
 
   ##
@@ -96,7 +95,7 @@ class MiGA::Daemon < MiGA::MiGA
     f_path = File.expand_path('daemon/status.json', project.path)
     return unless File.size? f_path
     say 'Loading previous status in daemon/status.json:'
-    status = JSON.parse(File.read(f_path), symbolize_names: true)
+    status = MiGA::Json.parse(f_path)
     status.keys.each do |i|
       status[i].map! do |j|
         j.tap do |k|
@@ -255,11 +254,6 @@ class MiGA::Daemon < MiGA::MiGA
   def terminate
     say 'Terminating daemon...'
     report_status
-    k = runopts(:kill)
-    @jobs_running.each do |i|
-      `#{k % i[:pid]}`
-      puts "Terminating pid:#{i[:pid]} for #{i[:task_name]}"
-    end
     f = File.expand_path('daemon/alive', project.path)
     File.unlink(f) if File.exist? f
   end

data/lib/miga/json.rb

@@ -0,0 +1,62 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'json'
+
+##
+# Taxonomic classifications in MiGA.
+class MiGA::Json < MiGA::MiGA
+
+  class << self
+
+    ##
+    # Default parsing options. Supported +opts+ keys:
+    # - +:contents+: If true, the input is assumed to be the contents to parse,
+    #   not the path to a JSON file.
+    # - +:default+: A base to attach the parsed hash. A Hash or a String (path).
+    # - +:additions+: If addition classes should be parsed. By default is false.
+    # - +:symbolize+: If names should be symbolized. By default it's true if
+    #   additions is false, or false otherwise. They can both be false, but an
+    #   exception will be raised if both are true
+    def default_opts(opts = {})
+      opts[:contents] ||= false
+      opts[:additions] ||= false
+      opts[:symbolize] = !opts[:additions] if opts[:symbolize].nil?
+      if opts[:additions] and opts[:symbolize]
+        raise 'JSON additions are not supported with symbolized names'
+      end
+      opts
+    end
+
+    ##
+    # Parse a JSON file in +path+ and return a hash. Optionally,
+    # use +default+ as the base to attach the parsed hash. +default+
+    # can be a Hash or a String (path). See +default_opts+ for supported
+    # +opts+.
+    def parse(path, opts = {})
+      opts = default_opts(opts)
+      cont = opts[:contents] ? path : File.read(path)
+      raise "Empty descriptor: #{opts[:contents] ? "''" : path}." if cont.empty?
+      y = JSON.parse(cont,
+        symbolize_names: opts[:symbolize],
+        create_additions: opts[:additions])
+      unless opts[:default].nil?
+        opts[:default] = parse(opts[:default]) if opts[:default].is_a? String
+        y.each{ |k, v| opts[:default][k] = v }
+        y = opts[:default]
+      end
+      y
+    end
+
+    ##
+    # Generates and returns prettyfied JSON to represent +obj+.
+    # If +path+ is passed, it saves the JSON in that file.
+    def generate(obj, path = nil)
+      y = JSON.pretty_generate(obj)
+      File.open(path, 'w') { |fh| fh.print y } unless path.nil?
+      y
+    end
+
+  end
+
+end

data/lib/miga/metadata.rb

@@ -57,14 +57,17 @@ class MiGA::Metadata < MiGA::MiGA
   def save
     MiGA.DEBUG "Metadata.save #{path}"
     self[:updated] = Time.now.to_s
-    json = JSON.pretty_generate(data)
+    json = MiGA::Json.generate(data)
     sleeper = 0.0
+    slept = 0
     while File.exist?(lock_file)
       sleeper += 0.1 if sleeper <= 10.0
       sleep(sleeper.to_i)
+      slept += sleeper.to_i
+      raise "Lock detected for over 10 minutes: #{lock_file}" if slept > 600
     end
     FileUtils.touch lock_file
-    ofh = File.open("#{path}.tmp", "w")
+    ofh = File.open("#{path}.tmp", 'w')
     ofh.puts json
     ofh.close
     raise "Lock-racing detected for #{path}." unless
@@ -81,9 +84,7 @@ class MiGA::Metadata < MiGA::MiGA
       sleeper += 0.1 if sleeper <= 10.0
       sleep(sleeper.to_i)
     end
-    # :symbolize_names does not play nicely with :create_additions
-    tmp = JSON.parse(File.read(path),
-      {:symbolize_names=>false, :create_additions=>true})
+    tmp = MiGA::Json.parse(path, additions: true)
     @data = {}
     tmp.each_pair{ |k,v| self[k] = v }
   end

data/lib/miga/project.rb

@@ -1,10 +1,10 @@
 # @package MiGA
 # @license Artistic-2.0
 
-require "miga/dataset"
-require "miga/project/result"
-require "miga/project/dataset"
-require "miga/project/plugins"
+require 'miga/dataset'
+require 'miga/project/result'
+require 'miga/project/dataset'
+require 'miga/project/plugins'
 
 ##
 # MiGA representation of a project.
@@ -17,16 +17,21 @@ class MiGA::Project < MiGA::MiGA
   ##
   # Absolute path to the project folder.
   attr_reader :path
-  
+
   ##
   # Information about the project as MiGA::Metadata.
   attr_reader :metadata
 
+  ##
+  # If true, it doesn't save changes
+  attr_accessor :do_not_save
+
   ##
   # Create a new MiGA::Project at +path+, if it doesn't exist and +update+ is
   # false, or load an existing one.
   def initialize(path, update=false)
     @datasets = {}
+    @do_not_save = false
     @path = File.absolute_path(path)
     self.create if not update and not Project.exist? self.path
     self.load if self.metadata.nil?
@@ -39,7 +44,7 @@ class MiGA::Project < MiGA::MiGA
   # Create an empty project.
   def create
     unless MiGA::MiGA.initialized?
-      raise "Impossible to create project in uninitialized MiGA."
+      raise 'Impossible to create project in uninitialized MiGA.'
     end
     dirs = [path] + @@FOLDERS.map{|d| "#{path}/#{d}" } +
       @@DATA_FOLDERS.map{ |d| "#{path}/data/#{d}"}
@@ -51,14 +56,20 @@ class MiGA::Project < MiGA::MiGA
         File.exist? "#{path}/daemon/daemon.json"
     self.load
   end
-  
+
   ##
-  # Save any changes persistently.
+  # Save any changes persistently. Do nothing if +do_not_save+ is true.
   def save
+    save! unless do_not_save
+  end
+
+  ##
+  # Save any changes persistently, regardless of +do_not_save+.
+  def save!
     metadata.save
     self.load
   end
-  
+
   ##
   # (Re-)load project data and metadata.
   def load
@@ -67,7 +78,7 @@ class MiGA::Project < MiGA::MiGA
     @metadata = MiGA::Metadata.load "#{path}/miga.project.json"
     raise "Couldn't find project metadata at #{path}" if metadata.nil?
   end
-  
+
   ##
   # Name of the project.
   def name ; metadata[:name] ; end
@@ -83,5 +94,5 @@ class MiGA::Project < MiGA::MiGA
   ##
   # Is this a project for multi-organism datasets?
   def is_multi? ; @@KNOWN_TYPES[type][:multi] ; end
-  
+
 end

data/lib/miga/remote_dataset/download.rb

@@ -91,7 +91,7 @@ class MiGA::RemoteDataset
     def ncbi_map(id, dbfrom, db)
       doc = download(:ncbi_map, dbfrom, id, :json, nil, [db])
       return if doc.empty?
-      tree = JSON.parse(doc, symbolize_names: true)
+      tree = MiGA::Json.parse(doc, contents: true)
       [:linksets, 0, :linksetdbs, 0, :links, 0].each do |i|
         tree = tree[i]
         break if tree.nil?

data/lib/miga/remote_dataset.rb

@@ -13,7 +13,9 @@ class MiGA::RemoteDataset < MiGA::MiGA
   class << self
     def ncbi_asm_acc2id(acc)
       return acc if acc =~ /^\d+$/
-      search_doc = JSON.parse download(:ncbi_search, :assembly, acc, :json)
+      search_doc = MiGA::Json.parse(
+        download(:ncbi_search, :assembly, acc, :json),
+        symbolize: false, contents: true)
       search_doc['esearchresult']['idlist'].first
     end
   end
@@ -55,6 +57,8 @@ class MiGA::RemoteDataset < MiGA::MiGA
   ##
   # Save dataset to the MiGA::Project +project+ identified with +name+. +is_ref+
   # indicates if it should be a reference dataset, and contains +metadata_def+.
+  # If +metadata_def+ includes +metadata_only: true+, no input data is
+  # downloaded.
   def save_to(project, name = nil, is_ref = true, metadata_def = {})
     name ||= ids.join('_').miga_name
     project = MiGA::Project.new(project) if project.is_a? String
@@ -63,16 +67,20 @@ class MiGA::RemoteDataset < MiGA::MiGA
     @metadata = get_metadata(metadata_def)
     udb = @@UNIVERSE[universe][:dbs][db]
     @metadata["#{universe}_#{db}"] = ids.join(',')
-    respond_to?("save_#{udb[:stage]}_to", true) or
-      raise "Unexpected error: Unsupported stage #{udb[:stage]} for #{db}."
-    send "save_#{udb[:stage]}_to", project, name, udb
+    unless @metadata[:metadata_only]
+      respond_to?("save_#{udb[:stage]}_to", true) or
+        raise "Unexpected error: Unsupported stage #{udb[:stage]} for #{db}."
+      send "save_#{udb[:stage]}_to", project, name, udb
+    end
     dataset = MiGA::Dataset.new(project, name, is_ref, metadata)
     project.add_dataset(dataset.name)
-    result = dataset.add_result(udb[:stage], true, is_clean: true)
-    result.nil? and
-      raise 'Empty dataset: seed result not added due to incomplete files.'
-    result.clean!
-    result.save
+    unless @metadata[:metadata_only]
+      result = dataset.add_result(udb[:stage], true, is_clean: true)
+      result.nil? and
+        raise 'Empty dataset: seed result not added due to incomplete files.'
+      result.clean!
+      result.save
+    end
     dataset
   end
 
@@ -141,8 +149,9 @@ class MiGA::RemoteDataset < MiGA::MiGA
     metadata[:ncbi_asm] ||= ids.first if universe == :ncbi and db == :assembly
     return nil unless metadata[:ncbi_asm]
     ncbi_asm_id = self.class.ncbi_asm_acc2id metadata[:ncbi_asm]
-    doc = JSON.parse(
-      self.class.download(:ncbi_summary, :assembly, ncbi_asm_id, :json))
+    doc = MiGA::Json.parse(
+      self.class.download(:ncbi_summary, :assembly, ncbi_asm_id, :json),
+      symbolize: false, contents: true)
     @_ncbi_asm_json_doc = doc['result'][ doc['result']['uids'].first ]
   end
 

data/lib/miga/result.rb

@@ -122,19 +122,14 @@ class MiGA::Result < MiGA::MiGA
       @data[:started] = File.read(s).chomp
       File.unlink s
     end
-    json = JSON.pretty_generate data
-    ofh = File.open(path, "w")
-    ofh.puts json
-    ofh.close
+    MiGA::Json.generate(data, path)
     self.load
   end
 
   ##
   # Load (or reload) result data in the JSON file #path.
   def load
-    json = File.read(path)
-    raise "Impossible to load result, empty descriptor: #{path}." if json.empty?
-    @data = JSON.parse(json, {:symbolize_names=>true})
+    @data = MiGA::Json.parse(path)
     @data[:files] ||= {}
     @results = (self[:results] || []).map{ |rs| MiGA::Result.new rs }
   end

data/lib/miga/tax_index.rb

@@ -57,8 +57,8 @@ class MiGA::TaxIndex < MiGA::MiGA
   ##
   # Generate JSON String for the index.
   def to_json
-    JSON.generate({ root:root.to_hash,
-      datasets:datasets.map{ |d| d.name } })
+    MiGA::Json.generate(
+      { root: root.to_hash, datasets: datasets.map{ |d| d.name } })
   end
 
   ##

data/lib/miga/taxonomy.rb

@@ -15,37 +15,37 @@ class MiGA::Taxonomy < MiGA::MiGA
   ##
   # Long names of the cannonical ranks.
   def self.LONG_RANKS() @@LONG_RANKS ; end
-  @@LONG_RANKS = {root: "root", ns: "namespace", d: "domain", k: "kingdom",
-    p: "phylum", c: "class", o: "order", f: "family", g: "genus", s: "species",
-    ssp: "subspecies", str: "strain", ds: "dataset"}
+  @@LONG_RANKS = {root: 'root', ns: 'namespace', d: 'domain', k: 'kingdom',
+    p: 'phylum', c: 'class', o: 'order', f: 'family', g: 'genus', s: 'species',
+    ssp: 'subspecies', str: 'strain', ds: 'dataset'}
 
   ##
   # Synonms for cannonical ranks.
   @@RANK_SYNONYMS = {
-    "namespace"=>"ns",
-    "domain"=>"d","superkingdom"=>"d",
-    "kingdom"=>"k",
-    "phylum"=>"p",
-    "class"=>"c",
-    "order"=>"o",
-    "family"=>"f",
-    "genus"=>"g",
-    "species"=>"s","sp"=>"s",
-    "subspecies"=>"ssp",
-    "strain"=>"str","isolate"=>"str","culture"=>"str",
-    "dataset"=>"ds","organism"=>"ds","genome"=>"ds","specimen"=>"ds"
+    'namespace' => 'ns',
+    'domain' => 'd', 'superkingdom' => 'd',
+    'kingdom' => 'k',
+    'phylum' => 'p',
+    'class' => 'c',
+    'order' => 'o',
+    'family' => 'f',
+    'genus' => 'g',
+    'species' => 's', 'sp' => 's',
+    'subspecies' => 'ssp',
+    'strain' => 'str', 'isolate' => 'str', 'culture' => 'str',
+    'dataset' => 'ds', 'organism' => 'ds', 'genome' => 'ds', 'specimen' => 'ds'
   }
 
   ##
   # Initialize from JSON-derived Hash +o+.
-  def self.json_create(o) new(o["str"]) ; end
+  def self.json_create(o) new(o['str']) ; end
 
   ##
   # Returns cannonical rank (Symbol) for the +rank+ String.
   def self.normalize_rank(rank)
     return rank.to_sym if @@_KNOWN_RANKS_H[rank.to_sym]
     rank = rank.to_s.downcase
-    return nil if rank=="no rank"
+    return nil if rank == 'no rank'
     rank = @@RANK_SYNONYMS[rank] unless @@RANK_SYNONYMS[rank].nil?
     rank = rank.to_sym
     return nil unless @@_KNOWN_RANKS_H[rank]
@@ -64,7 +64,7 @@ class MiGA::Taxonomy < MiGA::MiGA
   # either a rank:value pair (if +ranks+ is nil), or just values in the same
   # order as ther ranks in +ranks+. Alternatively, +str+ as a Hash with rank =>
   # value pairs is also supported.
-  def initialize(str, ranks=nil)
+  def initialize(str, ranks = nil)
     @ranks = {}
     if ranks.nil?
       case str when Array, Hash

data/lib/miga/version.rb

@@ -10,7 +10,7 @@ module MiGA
   # - Float representing the major.minor version.
   # - Integer representing gem releases of the current version.
   # - Integer representing minor changes that require new version number.
-  VERSION = [0.3, 9, 1]
+  VERSION = [0.3, 10, 0]
   
   ##
   # Nickname for the current major.minor version.
@@ -18,7 +18,7 @@ module MiGA
   
   ##
   # Date of the current gem release.
-  VERSION_DATE = Date.new(2019, 03, 8)
+  VERSION_DATE = Date.new(2019, 04, 9)
   
   ##
   # Reference of MiGA.

data/test/daemon_test.rb

@@ -114,11 +114,6 @@ class DaemonTest < Test::Unit::TestCase
       $d1.runopts(:latency, "!")
     end
     assert_equal("bash", $d1.runopts(:type))
-    assert_equal("kill -9 '%s'", $d1.runopts(:kill))
-    $d1.runopts(:type, "qsub")
-    assert_equal("qdel '%s'", $d1.runopts(:kill))
-    $d1.runopts(:type, "msub")
-    assert_equal("canceljob '%s'", $d1.runopts(:kill))
   end
 
   def test_say

data/test/json_test.rb

@@ -0,0 +1,37 @@
+require 'test_helper'
+
+class JsonTest < Test::Unit::TestCase
+  
+  def test_parse
+    assert_equal({a: 1, b: 2},
+      MiGA::Json.parse('{"a": 1, "b": 2}', contents: true))
+    assert_equal({'a' => 1, 'b' => 2},
+      MiGA::Json.parse('{"a": 1, "b": 2}', contents: true, symbolize: false))
+    assert_equal(1.0, MiGA::Json.parse('1.0', contents: true))
+  end
+
+  def test_defaults
+    tmp1 = Tempfile.new('test-parse-1.json')
+    tmp1.puts '{"a": 123, "k": false, "t": null}'
+    tmp1.close
+    assert_equal({a: 123, k: false, t: nil}, MiGA::Json.parse(tmp1.path))
+
+    tmp2 = Tempfile.new('test-parse-2.json')
+    tmp2.puts '{"a": 456, "kb": false, "t": 10.0}'
+    tmp2.close
+    assert_equal({a: 456, kb: false, t: 10.0}, MiGA::Json.parse(tmp2.path))
+
+    assert_equal(
+      {a: 123, k: false, kb: false, t: nil},
+      MiGA::Json.parse(tmp1.path, default: tmp2.path))
+    assert_equal(
+      {a: 456, k: false, kb: false, t: 10.0},
+      MiGA::Json.parse(tmp2.path, default: tmp1.path))
+  end
+
+  def test_generate
+    assert_equal("{\n  \"a\": 1,\n  \"b\": 2\n}",
+      MiGA::Json.generate({a: 1, b: 2}))
+  end
+
+end

data/utils/cleanup-databases.rb

@@ -5,17 +5,22 @@ require 'miga'
 
 ARGV[1] or abort "Usage: #{$0} path/to/project threads"
 
-$stderr.puts "Cleaning databases..."
+$stderr.puts 'Cleaning databases...'
 p = MiGA::Project.load(ARGV[0])
 ds_names = p.dataset_names
 thr = ARGV[1].to_i
 
+pc = [0] + (1 .. 100).map{ |i| ds_names.size * i / 100 }
+$stderr.puts (('.'*9 + '|')*10) + ' 100%'
+
 (0 .. thr-1).each do |t|
   fork do
-    k = -1
-    ds_names.each do |i|
-      k = (k+1) % thr
-      next unless k == t
+    ds_names.each_with_index do |i, idx|
+      while t == 0 and idx+1 > pc.first
+        $stderr.print '#'
+        pc.shift
+      end
+      next unless (idx % thr) == t
       d = p.dataset(i)
       next unless d.is_ref? and d.is_active?
       d.cleanup_distances!
@@ -23,4 +28,5 @@ thr = ARGV[1].to_i
   end
 end
 Process.waitall
+$stderr.puts ' Done'
 

data/utils/distance/runner.rb

@@ -108,7 +108,7 @@ class MiGA::DistanceRunner
 
     # Calculate all the AAIs/ANIs against the closest ANI95-clade (if AAI > 80%)
     cl_path = res.file_path :clades_ani95
-    if File.size? cl_path and tsk[0] == :clade_finding
+    if !cl_path.nil? and File.size? cl_path and tsk[0] == :clade_finding
       File.foreach(cl_path).
         map  { |i| i.chomp.split(',') }.
         find( lambda{[]} ){ |i| i.include? closest[:ds] }.

data/utils/enveomics/Pipelines/assembly.pbs/FastA.N50.pl

@@ -1 +1 @@
-utils/enveomics/Pipelines/assembly.pbs/../../Scripts/FastA.N50.pl
+../../Scripts/FastA.N50.pl

data/utils/enveomics/Pipelines/assembly.pbs/FastA.filterN.pl

@@ -1 +1 @@
-utils/enveomics/Pipelines/assembly.pbs/../../Scripts/FastA.filterN.pl
+../../Scripts/FastA.filterN.pl

data/utils/enveomics/Pipelines/assembly.pbs/FastA.length.pl

@@ -1 +1 @@
-utils/enveomics/Pipelines/assembly.pbs/../../Scripts/FastA.length.pl
+../../Scripts/FastA.length.pl

data/utils/enveomics/Pipelines/blast.pbs/FastA.split.pl

@@ -1 +1 @@
-utils/enveomics/Pipelines/blast.pbs/../../Scripts/FastA.split.pl
+../../Scripts/FastA.split.pl

data/utils/enveomics/Scripts/lib/enveomics.R

@@ -1 +1 @@
-utils/enveomics/Scripts/lib/../../enveomics.R
+../../enveomics.R

data/utils/ref-tree.R

@@ -7,15 +7,23 @@
 #= Load stuff
 argv <- commandArgs(trailingOnly=T)
 suppressPackageStartupMessages(library(ape))
-suppressPackageStartupMessages(library(phytools))
-suppressPackageStartupMessages(library(phangorn))
 suppressPackageStartupMessages(library(enveomics.R))
+inst <- c("phangorn", "phytools") %in% rownames(installed.packages())
+if(inst[1]){
+  suppressPackageStartupMessages(library(phangorn))
+  reroot.fun <- midpoint
+}else if(inst[2]){
+  suppressPackageStartupMessages(library(phytools))
+  reroot.fun <- midpoint.root
+}else{
+  reroot.fun <- function(x) return(x)
+}
 
 #= Main function
 ref_tree <- function(ani_file, out_base, q_dataset) {
   a <- read.table(ani_file, sep="\t", header=TRUE, as.is=TRUE)
   ani.d <- enve.df2dist(a[,1:3], default.d=0.9, max.sim=100)
-  ani.ph <- midpoint(bionj(ani.d))
+  ani.ph <- reroot.fun(bionj(ani.d))
   write.tree(ani.ph, paste(out_base, ".nwk", sep=""))
   pdf(paste(out_base, ".nwk.pdf", sep=""), 7, 7)
   plot(ani.ph, cex=1/3, type='fan',
@@ -24,42 +32,6 @@ ref_tree <- function(ani_file, out_base, q_dataset) {
   dev.off()
 }
 
-# Ancilliary functions
-midpoint <- function(tree){
-  dm = cophenetic(tree)
-  tree = unroot(tree)
-  rn = max(tree$edge)+1
-  maxdm = max(dm)
-  ind = which(dm==maxdm,arr=TRUE)[1,]
-  tmproot = Ancestors(tree, ind[1], "parent")
-  tree = phangorn:::reroot(tree, tmproot)
-  edge = tree$edge
-  el = tree$edge.length
-  children = tree$edge[,2]
-  left = match(ind[1], children)
-  tmp = Ancestors(tree, ind[2], "all")
-  tmp= c(ind[2], tmp[-length(tmp)])
-  right = match(tmp, children)
-  if(el[left]>= (maxdm/2)){
-    edge = rbind(edge, c(rn, ind[1]))
-    edge[left,2] = rn
-    el[left] = el[left] - (maxdm/2)
-    el = c(el, maxdm/2)
-  }else{
-    sel = cumsum(el[right])
-    i = which(sel>(maxdm/2))[1]
-    edge = rbind(edge, c(rn, tmp[i]))
-    edge[right[i],2] = rn
-    eltmp = sel[i] - (maxdm/2)
-    el = c(el, el[right[i]] - eltmp)
-    el[right[i]] = eltmp
-  }
-  tree$edge.length = el
-  tree$edge=edge
-  tree$Nnode = tree$Nnode+1
-  phangorn:::reorderPruning(phangorn:::reroot(tree, rn))
-}
-
 #= Main
 ref_tree(ani_file=argv[1], out_base=argv[2], q_dataset=argv[3])
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: miga-base
 version: !ruby/object:Gem::Version
-  version: 0.3.9.1
+  version: 0.3.10.0
 platform: ruby
 authors:
 - Luis M. Rodriguez-R
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-03-08 00:00:00.000000000 Z
+date: 2019-04-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons
@@ -142,6 +142,7 @@ files:
 - lib/miga/dataset.rb
 - lib/miga/dataset/base.rb
 - lib/miga/dataset/result.rb
+- lib/miga/json.rb
 - lib/miga/metadata.rb
 - lib/miga/project.rb
 - lib/miga/project/base.rb
@@ -184,6 +185,7 @@ files:
 - test/common_test.rb
 - test/daemon_test.rb
 - test/dataset_test.rb
+- test/json_test.rb
 - test/metadata_test.rb
 - test/project_test.rb
 - test/remote_dataset_test.rb
@@ -506,8 +508,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.7
+rubygems_version: 3.0.2
 signing_key: 
 specification_version: 4
 summary: MiGA