miga-base 0.4.1.0 → 0.4.2.0

data/lib/miga/cli/action/init.rb

@@ -0,0 +1,343 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'miga/cli/action'
+require 'shellwords'
+
+class MiGA::Cli::Action::Init < MiGA::Cli::Action
+
+  def parse_cli
+    cli.interactive = true
+    cli.defaults = {mytaxa: nil,
+      config: File.expand_path('.miga_modules', ENV['HOME']),
+      ask: false, auto: false, dtype: :bash}
+    cli.parse do |opt|
+      opt.on(
+        '-c', '--config PATH',
+        'Path to the Bash configuration file',
+        "By default: #{cli[:config]}"
+        ){ |v| cli[:config] = v }
+      opt.on(
+        '--[no-]mytaxa',
+        'Should I try setting up MyTaxa its dependencies?',
+        'By default: interactive (true if --auto)'
+        ){ |v| cli[:mytaxa] = v }
+      opt.on(
+        '--daemon-type STRING',
+        'Type of daemon launcher, one of: bash, qsub, msub, slurm',
+        "By default: interactive (#{cli[:dtype]} if --auto)"
+        ){ |v| cli[:dtype] = v.to_sym }
+      opt.on(
+        '--ask-all',
+        'Ask for the location of all software',
+        'By default, only the locations missing in PATH are requested'
+        ){ |v| cli[:ask] = v }
+    end
+  end
+
+  def perform
+    miga = MiGA.root_path
+    cli.puts <<BANNER
+===[ Welcome to MiGA, the Microbial Genome Atlas ]===
+
+I'm the initialization script, and I'll sniff around your computer to
+make sure you have all the requirements for MiGA data processing.
+
+BANNER
+
+    if cli.ask_user(
+          'Would you like to see all the requirements before starting?',
+          'no', %w(yes no)) == 'yes'
+      cli.puts ''
+      File.open(File.expand_path('utils/requirements.txt', miga), 'r') do |fh|
+        fh.each_line { |ln| cli.puts ln }
+      end
+      cli.puts ''
+    end
+
+    rc_path = File.expand_path('.miga_rc', ENV['HOME'])
+    if File.exist? rc_path
+      if cli.ask_user(
+            'I found a previous configuration. Do you want to continue?',
+            'yes', %w(yes no)) == 'no'
+        cli.puts 'OK, see you soon!'
+        exit(0)
+      end
+    end
+    rc_fh = File.open(rc_path, 'w')
+    rc_fh.puts <<BASH
+#!/bin/bash
+# `miga init` made this on #{Time.now}
+
+BASH
+
+    # Check bash configuration file
+    unless File.exist? cli[:config]
+      cli[:config] = cli.ask_user(
+            'Is there a script I need to load at startup?',
+            cli[:config])
+    end
+    if File.exist? cli[:config]
+      cli[:config] = File.expand_path(cli[:config])
+      cli.puts "Found bash configuration script: #{cli[:config]}"
+      rc_fh.puts "MIGA_STARTUP='#{cli[:config]}'"
+      rc_fh.puts '. "$MIGA_STARTUP"'
+    else
+      cli[:config] = '/dev/null'
+    end
+    cli.puts ''
+
+    # Check for software requirements
+    cli.puts 'Looking for requirements:'
+    if cli[:mytaxa].nil?
+      cli[:mytaxa] = cli.ask_user(
+            'Should I include MyTaxa modules?',
+            'yes', %w(yes no)) == 'yes'
+    end
+    rc_fh.puts 'export MIGA_MYTAXA="no"' unless cli[:mytaxa]
+    paths = {}
+    rc_fh.puts 'MIGA_PATH=""'
+    File.open(File.expand_path('utils/requirements.txt', miga), 'r') do |fh|
+      fh.each_line do |ln|
+        next if $. < 3
+        r = ln.chomp.split(/\t+/)
+        next if r[0] =~ /\(opt\)$/ && !cli[:mytaxa]
+        cli.print "Testing #{r[0]}#{" (#{r[3]})" if r[3]}... "
+        path = nil
+        loop do
+          d_path = File.dirname(run_cmd(cli, "which #{r[1].shellescape}"))
+          if cli[:ask] || d_path == '.'
+            path = cli.ask_user('Where can I find it?', d_path, nil, true)
+          else
+            path = d_path
+            cli.puts path
+          end
+          if File.executable?(File.expand_path(r[1], path))
+            if d_path != path
+              rc_fh.puts "MIGA_PATH=\"#{path}:$MIGA_PATH\" # #{r[1]}"
+            end
+            break
+          end
+          cli.print "I cannot find #{r[1]} "
+        end
+        paths[r[1]] = File.expand_path(r[1], path).shellescape
+      end
+    end
+    rc_fh.puts 'export PATH="$MIGA_PATH$PATH"'
+    cli.puts ''
+
+    # Check for other files
+    if cli[:mytaxa]
+      cli.puts 'Looking for MyTaxa databases:'
+      mt = File.dirname paths["MyTaxa"]
+      cli.print 'Looking for scores... '
+      unless Dir.exist?(File.expand_path('db', mt))
+        cli.puts "no.\nExecute 'python2 #{mt}/utils/download_db.py'."
+        exit(1)
+      end
+      cli.puts 'yes.'
+      cli.print 'Looking for diamond db... '
+      unless File.exist?(File.expand_path('AllGenomes.faa.dmnd', mt))
+        cli.puts "no.\nDownload " \
+          "'http://enve-omics.ce.gatech.edu/data/public_mytaxa/" \
+          "AllGenomes.faa.dmnd' into #{mt}."
+        exit(1)
+      end
+      cli.puts ''
+    end
+
+    # Check for R packages
+    cli.puts 'Looking for R packages:'
+    %w(enveomics.R ape cluster vegan).each do |pkg|
+      cli.print "Testing #{pkg}... "
+      if test_r_package(cli, paths, pkg)
+        cli.puts 'yes.'
+      else
+        cli.puts 'no, installing'
+        cli.print '' + install_r_package(cli, paths, pkg)
+        unless test_r_package(cli, paths, pkg)
+          raise "Unable to auto-install R package: #{pkg}"
+        end
+      end
+    end
+    cli.puts ''
+
+    # Check for Ruby gems
+    cli.puts 'Looking for Ruby gems:'
+    %w(sqlite3 daemons json).each do |pkg|
+      cli.print "Testing #{pkg}... "
+      if test_ruby_gem(cli, paths, pkg)
+        cli.puts 'yes.'
+      else
+        cli.puts 'no, installing'
+        # This hackey mess is meant to ensure the test and installation are done
+        # on the configuration Ruby, not on the Ruby currently executing the init
+        # action
+        cli.print install_ruby_gem(cli, paths, pkg)
+        unless test_ruby_gem(cli, paths, pkg)
+          raise "Unable to auto-install Ruby gem: #{pkg}"
+        end
+      end
+    end
+    cli.puts ''
+
+    # Configure daemon
+    cli.puts 'Default daemon configuration:'
+    daemon_f = File.expand_path('.miga_daemon.json', ENV['HOME'])
+    unless File.exist?(daemon_f) and cli.ask_user(
+              'A template daemon already exists, do you want to preserve it?',
+              'yes', %w(yes no)) == 'yes'
+      v = {created: Time.now.to_s, updated: Time.now.to_s}
+      v[:type] = cli.ask_user(
+        'Please select the type of daemon you want to setup',
+        cli[:dtype], %w(bash qsub msub slurm))
+      case v[:type]
+        when 'bash'
+          v[:latency] = cli.ask_user(
+            'How long should I sleep? (in seconds)', '30').to_i
+          v[:maxjobs] = cli.ask_user(
+            'How many jobs can I launch at once?', '6').to_i
+          v[:ppn]     = cli.ask_user(
+            'How many CPUs can I use per job?', '2').to_i
+          cli.puts 'Setting up internal daemon defaults.'
+          cli.puts 'If you don\'t understand this just leave default values:'
+          v[:cmd]     = cli.ask_user(
+            "How should I launch tasks?\n  %1$s: script path, " \
+              "%2$s: variables, %3$d: CPUs, %4$s: log file, %5$s: task name.\n",
+            "%2$s '%1$s' > '%4$s' 2>&1")
+          v[:var]     = cli.ask_user(
+            "How should I pass variables?\n  %1$s: keys, %2$s: values.\n",
+            "%1$s=%2$s")
+          v[:varsep]  = cli.ask_user(
+            'What should I use to separate variables?', ' ')
+          v[:alive]   = cli.ask_user(
+            "How can I know that a process is still alive?\n  %1$s: PID, " \
+              "output should be 1 for running and 0 for non-running.\n",
+            "ps -p '%1$s'|tail -n+2|wc -l")
+          v[:kill]    = cli.ask_user(
+            "How should I terminate tasks?\n  %s: process ID.", "kill -9 '%s'")
+        when 'slurm'
+          queue       = cli.ask_user(
+            'What queue should I use?', nil, nil, true)
+          v[:latency] = cli.ask_user(
+            'How long should I sleep? (in seconds)', '150').to_i
+          v[:maxjobs] = cli.ask_user(
+            'How many jobs can I launch at once?', '300').to_i
+          v[:ppn]     = cli.ask_user(
+            'How many CPUs can I use per job?', '2').to_i
+          cli.puts 'Setting up internal daemon defaults'
+          cli.puts 'If you don\'t understand this just leave default values:'
+          v[:cmd]     = cli.ask_user(
+            "How should I launch tasks?\n  %1$s: script path, " \
+              "%2$s: variables, %3$d: CPUs, %4$d: log file, %5$s: task name.\n",
+            "%2$s sbatch --partition='#{queue}' --export=ALL " \
+              "--nodes=1 --ntasks-per-node=%3$d --output='%4$s' " \
+              "--job-name='%5$s' --mem=9G --time=12:00:00 %1$s " \
+              "| perl -pe 's/.* //'")
+          v[:var]     = cli.ask_user(
+            "How should I pass variables?\n  %1$s: keys, %2$s: values.\n",
+            "%1$s=%2$s")
+          v[:varsep]  = cli.ask_user(
+            'What should I use to separate variables?', ' ')
+          v[:alive]   = cli.ask_user(
+            "How can I know that a process is still alive?\n  %1$s: job id, " \
+              "output should be 1 for running and 0 for non-running.\n",
+            "squeue -h -o %%t -j '%1$s' | grep '^PD\\|R\\|CF\\|CG$' " \
+              "| tail -n 1 | wc -l")
+          v[:kill]    = cli.ask_user(
+            "How should I terminate tasks?\n  %s: process ID.", "scancel '%s'")
+        else # [qm]sub
+          queue       = cli.ask_user('What queue should I use?', nil, nil, true)
+          v[:latency] = cli.ask_user(
+            'How long should I sleep? (in seconds)', '150').to_i
+          v[:maxjobs] = cli.ask_user(
+            'How many jobs can I launch at once?', '300').to_i
+          v[:ppn]     = cli.ask_user(
+            'How many CPUs can I use per job?', '2').to_i
+          cli.puts 'Setting up internal daemon defaults.'
+          cli.puts 'If you don\'t understand this just leave default values:'
+          v[:cmd]     = cli.ask_user(
+            "How should I launch tasks?\n  %1$s: script path, " \
+              "%2$s: variables, %3$d: CPUs, %4$d: log file, %5$s: task name.\n",
+            "#{v[:type]} -q '#{queue}' -v '%2$s' -l nodes=1:ppn=%3$d %1$s " \
+              "-j oe -o '%4$s' -N '%5$s' -l mem=9g -l walltime=12:00:00 " \
+              "| grep .")
+          v[:var]     = cli.ask_user(
+            "How should I pass variables?\n  %1$s: keys, %2$s: values.\n",
+            "%1$s=%2$s")
+          v[:varsep]  = cli.ask_user(
+            'What should I use to separate variables?', ',')
+          if v[:type] == 'qsub'
+            v[:alive] = cli.ask_user(
+              "How can I know that a process is still alive?\n  " \
+                "%1$s: job id, output should be 1 for running and " \
+                "0 for non-running.\n",
+              "qstat -f '%1$s'|grep ' job_state ='|perl -pe 's/.*= //'" \
+                "|grep '[^C]'|tail -n1|wc -l|awk '{print $1}'")
+            v[:kill]  = cli.ask_user(
+              "How should I terminate tasks?\n  %s: process ID.", "qdel '%s'")
+          else # msub
+            v[:alive] = cli.ask_user(
+              "How can I know that a process is still alive?\n  " \
+                "%1$s: job id, output should be 1 for running and " \
+                "0 for non-running.\n",
+              "checkjob '%1$s'|grep '^State:'|perl -pe 's/.*: //'" \
+                "|grep 'Deferred\\|Hold\\|Idle\\|Starting\\|Running\\|Blocked'" \
+                "|tail -n1|wc -l|awk '{print $1}'")
+            v[:kill]  = cli.ask_user(
+              "How should I terminate tasks?\n  %s: process ID.",
+              "canceljob '%s'")
+          end
+      end
+      File.open(daemon_f, 'w') { |fh| fh.puts JSON.pretty_generate(v) }
+    end
+    cli.puts ''
+
+    rc_fh.puts <<FOOT
+
+MIGA_CONFIG_VERSION='#{MiGA::MiGA.VERSION}'
+MIGA_CONFIG_LONGVERSION='#{MiGA::MiGA.LONG_VERSION}'
+MIGA_CONFIG_DATE='#{Time.now}'
+
+FOOT
+
+    cli.puts 'Configuration complete. MiGA is ready to work!'
+    cli.puts ''
+
+  end
+
+  def empty_action
+  end
+
+  def run_cmd(cli, cmd)
+    `. "#{cli[:config]}" && #{cmd}`
+  end
+
+  def run_r_cmd(cli, paths, cmd)
+    run_cmd(cli,
+      "echo #{cmd.shellescape} | #{paths['R'].shellescape} --vanilla -q 2>&1")
+  end
+
+  def test_r_package(cli, paths, pkg)
+    run_r_cmd(cli, paths, "library('#{pkg}')")
+    $?.success?
+  end
+
+  def install_r_package(cli, paths, pkg)
+    r_cmd = "install.packages('#{pkg}', repos='http://cran.rstudio.com/')"
+    run_r_cmd(cli, paths, r_cmd)
+  end
+
+  def test_ruby_gem(cli, paths, pkg)
+    run_cmd(cli,
+      "#{paths['ruby'].shellescape} -r #{pkg.shellescape} -e '' 2>/dev/null")
+    $?.success?
+  end
+
+  def install_ruby_gem(cli, paths, pkg)
+    gem_cmd = "Gem::GemRunner.new.run %w(install --user #{pkg})"
+    run_cmd(cli, "#{paths['ruby'].shellescape} \
+        -r rubygems -r rubygems/gem_runner \
+        -e #{gem_cmd.shellescape} 2>&1")
+  end
+end

data/lib/miga/cli/action/ln.rb

@@ -0,0 +1,42 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'miga/cli/action'
+
+class MiGA::Cli::Action::Ln < MiGA::Cli::Action
+
+  def parse_cli
+    cli.defaults = {info: false, force: false, method: :hardlink}
+    cli.parse do |opt|
+      cli.opt_object(opt, [:project, :dataset_opt])
+      opt.on(
+        '-Q', '--project-target PATH',
+        '(Mandatory) Path to the project where to link the dataset'
+        ){ |v| cli[:project2] = v }
+      opt.on(
+        '-f', '--force',
+        'Force linking, even if dataset\'s preprocessing is incomplete'
+        ){ |v| cli[:force] = v }
+      opt.on(
+        '-s', '--symlink',
+        'Create symlinks instead of the default hard links'
+        ){ cli[:method] = :symlink }
+      opt.on(
+        '-c', '--copy',
+        'Create copies instead of the default hard links'
+        ){ cli[:method] = :copy }
+      cli.opt_filter_datasets(opt)
+    end
+  end
+
+  def perform
+    p = cli.load_project
+    q = cli.load_project(:project2, '-Q')
+    ds = cli.load_and_filter_datasets
+    ds.each do |d|
+      next unless cli[:force] or d.done_preprocessing?
+      cli.puts d.name
+      q.import_dataset(d, cli[:method])
+    end
+  end
+end

data/lib/miga/cli/action/ls.rb

@@ -0,0 +1,55 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'miga/cli/action'
+
+class MiGA::Cli::Action::Ls < MiGA::Cli::Action
+
+  def parse_cli
+    cli.defaults = {info: false, processing: false, silent: false}
+    cli.parse do |opt|
+      cli.opt_object(opt, [:project, :dataset_opt])
+      cli.opt_filter_datasets(opt)
+      opt.on(
+        '-i', '--info',
+        'Print additional information on each dataset'
+        ){ |v| cli[:info] = v }
+      opt.on(
+        '-p', '--processing',
+        'Print information on processing advance'
+        ){ |v| cli[:processing] = v }
+      opt.on(
+        '-m', '--metadata STRING',
+        'Print name and metadata field only',
+        'If set, ignores -i and assumes --tab'
+        ){ |v| cli[:datum] = v }
+      opt.on(
+        '--tab',
+        'Return a tab-delimited table'
+        ){ |v| cli[:tabular] = v }
+      opt.on(
+        '-s', '--silent',
+        'No output and exit with non-zero status if the dataset list is empty'
+        ){ |v| cli[:silent] = v }
+    end
+  end
+
+  def perform
+    ds = cli.load_and_filter_datasets(cli[:silent])
+    exit(ds.empty? ? 1 : 0) if cli[:silent]
+    if !cli[:datum].nil?
+      ds.each do |d|
+        v = d.metadata[cli[:datum]]
+        puts "#{d.name}\t#{v.nil? ? '?' : v}"
+      end
+    elsif cli[:info]
+      cli.table(Dataset.INFO_FIELDS, ds.map { |d| d.info })
+    elsif cli[:processing]
+      comp = %w[- done queued]
+      cli.table([:name] + MiGA::Dataset.PREPROCESSING_TASKS,
+        ds.map { |d| [d.name] + d.profile_advance.map { |i| comp[i] } })
+    else
+      ds.each { |d| cli.puts d.name }
+    end
+  end
+end

data/lib/miga/cli/action/ncbi_get.rb

@@ -0,0 +1,218 @@
+# @package MiGA
+# @license Artistic-2.0
+
+require 'miga/cli/action'
+require 'miga/remote_dataset'
+require 'csv'
+
+class MiGA::Cli::Action::NcbiGet < MiGA::Cli::Action
+
+  def parse_cli
+    cli.defaults = {query: false, unlink: false,
+      reference: false, legacy_name: false,
+      complete: false, chromosome: false,
+      scaffold: false, contig: false, add_version: true, dry: false,
+      get_md: false, only_md: false, save_every: 1}
+    cli.parse do |opt|
+      cli.opt_object(opt, [:project])
+      opt.on(
+        '-T', '--taxon STRING',
+        '(Mandatory unless --reference) Taxon name (e.g., a species binomial)'
+        ){ |v| cli[:taxon] = v }
+      opt.on('--reference',
+        'Download all reference genomes (ignore any other status)'
+        ){ |v| cli[:reference] = v }
+      opt.on(
+        '--complete',
+        'Download complete genomes'
+        ){ |v| cli[:complete] = v }
+      opt.on('--chromosome',
+        'Download complete chromosomes'
+        ){ |v| cli[:chromosome] = v }
+      opt.on(
+        '--scaffold',
+        'Download genomes in scaffolds'
+        ){ |v| cli[:scaffold] = v }
+      opt.on(
+        '--contig',
+        'Download genomes in contigs'
+        ){ |v| cli[:contig] = v }
+      opt.on(
+        '--all',
+        'Download all genomes (in any status)') do
+          cli[:complete] = true
+          cli[:chromosome] = true
+          cli[:scaffold] = true
+          cli[:contig] = true
+        end
+      opt.on(
+        '--no-version-name',
+        'Do not add sequence version to the dataset name',
+        'Only affects --complete and --chromosome'
+        ){ |v| cli[:add_version] = v }
+      opt.on(
+        '--legacy-name',
+        'Use dataset names based on chromosome entries instead of assembly'
+        ){ |v| cli[:legacy_name] = v }
+      opt.on('--blacklist PATH',
+        'A file with dataset names to blacklist'
+        ){ |v| cli[:blacklist] = v }
+      opt.on(
+        '--dry',
+        'Do not download or save the datasets'
+        ){ |v| cli[:dry] = v }
+      opt.on(
+        '--get-metadata',
+        'Only download and update metadata for existing datasets'
+        ){ |v| cli[:get_md] = v }
+      opt.on('--only-metadata',
+        'Create datasets without input data but retrieve all metadata'
+        ){ |v| cli[:only_md] = v }
+      opt.on(
+        '--save-every INT', Integer,
+        'Save project every this many downloaded datasets',
+        'If zero, it saves the project only once upon completion',
+        "By default: #{cli[:save_every]}"
+        ){ |v| cli[:save_every] = v }
+      opt.on(
+        '-q', '--query',
+        'Register the datasets as queries, not reference datasets'
+        ){ |v| cli[:query] = v }
+      opt.on(
+        '-u', '--unlink',
+        'Unlink all datasets in the project missing from the download list'
+        ){ |v| cli[:unlink] = v }
+      opt.on('-R', '--remote-list PATH',
+        'Path to an output file with the list of all datasets listed remotely'
+        ){ |v| cli[:remote_list] = v }
+      opt.on(
+        '--api-key STRING',
+        'NCBI API key'
+        ){ |v| ENV['NCBI_API_KEY'] = v }
+    end
+  end
+
+  def perform
+    cli.ensure_par(taxon: '-T') unless cli[:reference]
+    unless %w[reference complete chromosome scaffold contig].any?{ |i| cli[i.to_sym] }
+      raise 'No action requested: pick at least one type of genome'
+    end
+    cli[:save_every] = 1 if cli[:dry]
+
+    p = cli.load_project
+    d = []
+    ds = {}
+    downloaded = 0
+
+    url_base = 'https://www.ncbi.nlm.nih.gov/genomes/solr2txt.cgi?'
+    url_param = {
+      q: '[display()].' +
+        'from(GenomeAssemblies).' +
+        'usingschema(/schema/GenomeAssemblies).' +
+        'matching(tab==["Prokaryotes"] and q=="' + cli[:taxon].tr('"',"'") + '"',
+      fields: 'organism|organism,assembly|assembly,replicons|replicons,' +
+        'level|level,ftp_path_genbank|ftp_path_genbank,release_date|release_date,' +
+        'strain|strain',
+      nolimit: 'on',
+    }
+    if cli[:reference]
+      url_param[:q] += ' and refseq_category==["representative"]'
+    else
+      status = {
+        complete: 'Complete',
+        chromosome: ' Chromosome', # <- The leading space is *VERY* important!
+        scaffold: 'Scaffold',
+        contig: 'Contig'
+      }.map { |k, v| '"' + v + '"' if cli[k] }.compact.join(',')
+      url_param[:q] += ' and level==[' + status + ']'
+    end
+    url_param[:q] += ')'
+    url = url_base + URI.encode_www_form(url_param)
+    cli.say 'Downloading genome list'
+    lineno = 0
+    doc = RemoteDataset.download_url(url)
+    CSV.parse(doc, headers: true).each do |r|
+      asm = r['assembly']
+      next if asm.nil? or asm.empty? or asm == '-'
+      next unless r['ftp_path_genbank']
+
+      # Get replicons
+      rep = r['replicons'].nil? ? nil : r['replicons'].
+          split('; ').map{ |i| i.gsub(/.*:/,'') }.map{ |i| i.gsub(/\/.*/, '') }
+
+      # Set name
+      if cli[:legacy_name] and cli[:reference]
+        n = r['#organism'].miga_name
+      else
+        if cli[:legacy_name] and ['Complete',' Chromosome'].include? r['level']
+          acc = rep.nil? ? '' : rep.first
+        else
+          acc = asm
+        end
+        acc.gsub!(/\.\d+\Z/, '') unless cli[:add_version]
+        n = "#{r['#organism']}_#{acc}".miga_name
+      end
+
+      # Register for download
+      fna_url = r['ftp_path_genbank'] + '/' +
+        File.basename(r['ftp_path_genbank']) + '_genomic.fna.gz'
+      ds[n] = {
+        ids: [fna_url], db: :assembly_gz, universe: :web,
+        md: {
+          type: :genome, ncbi_asm: asm, strain: r['strain']
+        }
+      }
+      ds[n][:md][:ncbi_nuccore] = rep.join(',') unless rep.nil?
+      ds[n][:md][:release_date] =
+        Time.parse(r['release_date']).to_s unless r['release_date'].nil?
+    end
+
+    # Discard blacklisted
+    unless cli[:blacklist].nil?
+      cli.say "Discarding datasets in #{cli[:blacklist]}"
+      File.readlines(cli[:blacklist]).
+        select{ |i| i !~ /^#/ }.map(&:chomp).each{ |i| ds.delete i }
+    end
+
+    # Download entries
+    cli.say "Downloading #{ds.size} " + (ds.size == 1 ? 'entry' : 'entries')
+    p.do_not_save = true if cli[:save_every] != 1
+    ds.each do |name, body|
+      d << name
+      cli.puts name
+      next if p.dataset(name).nil? == cli[:get_md]
+      downloaded += 1
+      next if cli[:dry]
+      cli.say '  Locating remote dataset'
+      body[:md][:metadata_only] = true if cli[:only_md]
+      rd = RemoteDataset.new(body[:ids], body[:db], body[:universe])
+      if cli[:get_md]
+        cli.say '  Updating dataset'
+        rd.update_metadata(p.dataset(name), body[:md])
+      else
+        cli.say '  Creating dataset'
+        rd.save_to(p, name, !cli[:query], body[:md])
+        p.add_dataset(name)
+      end
+      p.save! if cli[:save_every] > 1 and (downloaded % cli[:save_every]) == 0
+    end
+
+    p.do_not_save = false
+    p.save! if cli[:save_every] != 1
+
+    # Finalize
+    cli.say "Datasets listed: #{d.size}"
+    cli.say "Datasets #{cli[:dry] ? 'to download' : 'downloaded'}: " +
+      downloaded.to_s
+    unless cli[:remote_list].nil?
+      File.open(cli[:remote_list], 'w') do |fh|
+        d.each { |i| fh.puts i }
+      end
+    end
+    if cli[:unlink]
+      unlink = p.dataset_names - d
+      unlink.each { |i| p.unlink_dataset(i).remove! }
+      cli.say "Datasets unlinked: #{unlink.size}"
+    end
+  end
+end