ncbi-blast-dbs 0.0.2 → 0.0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: 5e0cc1d52b816514b98df36ded99c565d83f7f16
-  data.tar.gz: a42179ad695241c4774e44e891ec90ad9d657218
+SHA256:
+  metadata.gz: bcce1b77f891ca2abb26a791de778fb9305d0e8a998f971eff97393e11f6ea87
+  data.tar.gz: 1dc5dc4d9f3d2bf0f6dea3b75bc04f1cefbece636598c941dcf08f9c2affba2a
 SHA512:
-  metadata.gz: 2038176b3f4a2f4f1407116465a104247ed0c51b221533a9ac3767e7addcbce854f5368894921d91b0d4e5848cb72d8ac8a986b124ce4f0d9351dcfd678e1726
-  data.tar.gz: d1b713b964f1f9ab68d757252a3f3400f89772665a6e36658dcc42e04c76994fd781c51aeebf5a209c9317b89e35228ee16f37ed90baf2ce044e04a13c12421b
+  metadata.gz: fb6f3996b8e344a7e98af27c6187bc71a57cdf3996ed74d84243355c6c12aa9e122081b31b2b006d995fea04011c698cde49ef692c8b8eb6b45a790b2490d17f
+  data.tar.gz: bc9f835b65e8bb3fd7923d697a9b98096f2d5b5b8e9f7e5c0482f0996aece512397fecc70db78426e93bb91b99a199f7b9cd20bb48c0f46b915e5e580b7f7ef5

data/.ruby-version

@@ -0,0 +1 @@
+1.9

data/README.md

@@ -1,14 +1,15 @@
 ## Fast download BLAST databases from NCBI
 
 Database files (volumes) are downloaded in parallel: number of threads to use
-is determined automatically. MD5 checksum is verified and the database files
-extracted upon download. Database volumes are not downloaded in a particular
-order. Downloads are incremental, that is it will only download new data,
-provided the `.tar.gz` files have not been deleted.
+is determined automatically. MD5 checksum is verified and the database volume
+extracted upon download.  Database volumes are not downloaded in a particular
+order. The volumes are updated if a newer version is available on the server,
+or re-downloaded if corrupt. Aborted downloads are safely resumed.
 
-It is faster than NCBI's `update_blastdb.pl`. But unlike `update_blastdb.pl`,
-which is a pure Perl script, it delegates download and checksum verification
-to `wget` and `md5sum` and is thus not as universal.
+`ncbi-blast-dbs` is faster than NCBI's `update_blastdb.pl`. But unlike
+`update_blastdb.pl`, which is a pure Perl script, `ncbi-blast-dbs` delegates
+download and checksum verification to `wget` and `md5sum` / `md5` and is thus
+not as universal.
 
 ### Installation
 
@@ -22,7 +23,10 @@ to `wget` and `md5sum` and is thus not as universal.
 
 #### Download all volumes of a BLAST database
 
-    ncbi-blast-dbs nr
+    ncbi-blast-dbs nt nr
+
+Databases are downloaded one after the other. Volumes of each database are
+downloaded in parallel. Downloads are placed in the current directory.
 
 NCBI expects users to submit their email address when downloading data from
 their FTP server. To comply with that, download as:

data/bin/ncbi-blast-dbs

@@ -1,5 +1,20 @@
 #!/usr/bin/env ruby
 
 require 'rake'
-import "#{__dir__}/../lib/ncbi-blast-dbs.rake"
-app = Rake.application; app.init; app.load_imports; app.top_level
+
+trap :INT do
+  puts "Quitting ..."
+  exit!
+end
+
+if ARGV.include? "http";
+  import "#{File.dirname(__FILE__)}/../lib/http-ncbi-blast-dbs.rake"
+  Rake.application.init 'http-ncbi-blast-dbs'
+  Rake.application.load_imports
+  Rake.application.top_level  
+else;
+  import "#{File.dirname(__FILE__)}/../lib/ncbi-blast-dbs.rake"
+  Rake.application.init 'ncbi-blast-dbs'
+  Rake.application.load_imports
+  Rake.application.top_level
+end

data/lib/http-ncbi-blast-dbs.rake

@@ -0,0 +1,84 @@
+require 'net/http'
+require 'uri'
+puts "using http-ncbi-dbs-dgs.rake"
+# Downloads tarball at the given URL if a local copy does not exist, or if the
+# local copy is older than at the given URL, or if the local copy is corrupt.
+def download(url, last_to_do)
+  file = File.basename(url)
+
+  # # Resume an interrupted download or fetch the file for the first time. If
+  # # the file on the server is newer, then it is downloaded from start.
+
+  sh "wget -Nc --no-verbose #{url}"
+  # If the local copy is already fully retrieved, then the previous command
+  # ignores the timestamp. So we check with the server again if the file on
+  # the server is newer and if so download the new copy.
+  sh "wget -N --no-verbose #{url}"
+  sh "wget -Nc --no-verbose #{url}.md5"
+  sh "wget -N --no-verbose #{url}.md5"
+  # Immediately download md5 and verify the tarball. Re-download tarball if
+  # corrupt; extract otherwise.
+  sh "md5sum -c #{file}.md5" do |matched, _|
+    if !matched
+      sh "rm #{file} #{file}.md5"; download(url)
+    # too many tar instances unzipping the same file clutter the system
+    elsif file == last_to_do;
+      sh "tar xfov #{file}"      
+    else
+      # at least nr and nt tarballs have identical files .?al; unsure of others
+      sh "tar xfov #{file} --exclude='*.?al' --exclude='taxdb*'"
+    end
+  end
+end
+
+
+def databases
+  method = 'https://'
+  host, dir = 'ftp.ncbi.nlm.nih.gov', 'blast/db'
+  uri = URI.parse(method + host + "/" + dir + "/")
+
+  response = Net::HTTP.get_response(uri)
+  body = response.body.split
+
+  array_of_files = []
+  body.each do |line|
+    # regex takes the raw http response, matches lines such as:
+    #    href="tsa_nt.06.tar.gz.md5">tsa_nt.06.tar.gz</a>
+    # Returns:
+    # tsa_nt.06.tar.gz
+    filenames_and_newlines = line[/(^href=".*">)(.*tar.gz|.*md5)(<\/a>)$/, 2]
+    array_of_files.append(filenames_and_newlines) unless filenames_and_newlines.nil?
+  end
+
+  # append the full path to file for downstream wget
+  array_of_files.map! { |string| "".concat("/blast/db/", string ) }
+  array_of_files.
+    map { |file| File.join(host, file) }.
+    select { |file| file.match(/\.tar\.gz$/) }.
+    group_by { |file| File.basename(file).split('.')[0] }
+end
+
+
+# Create user-facing task for each database to drive the download of its
+# volumes in parallel.
+databases.each do |name, files|
+  last = { name => files.last }
+  multitask(name => files.map { |file| task(file) { download(file, last.values.uniq) } })
+end
+
+# List name of all databases that can be downloaded if executed without
+# any arguments.
+task :default do
+  databases
+  puts databases.keys.push('taxdump').join(', ')
+end
+
+task :taxdump do
+  download('https://ftp.ncbi.nih.gov/pub/taxonomy/taxdump.tar.gz', "nil")
+end
+
+# Ruby being over my head, this is my quick-and-dirty way to trick it ignoring
+# "http" as a task rather than a specification. Happy for an expert to fix it up!
+task :http do
+  puts "using http method"
+end

data/lib/ncbi-blast-dbs.rake

@@ -1,12 +1,45 @@
 require 'net/ftp'
 
+# Downloads tarball at the given URL if a local copy does not exist, or if the
+# local copy is older than at the given URL, or if the local copy is corrupt.
 def download(url)
   file = File.basename(url)
-  sh "wget -Nc #{url} && wget -Nc #{url}.md5 &&"\
-     " md5sum -c #{file}.md5 && tar xvf #{file}"\
-     " || rm #{file} #{file}.md5"
+  # Resume an interrupted download or fetch the file for the first time. If
+  # the file on the server is newer, then it is downloaded from start.
+  sh "wget -Nc --no-verbose #{url}"
+  # If the local copy is already fully retrieved, then the previous command
+  # ignores the timestamp. So we check with the server again if the file on
+  # the server is newer and if so download the new copy.
+  sh "wget -N --no-verbose #{url}"
+
+  # Download Md5
+  sh "wget --no-verbose #{url}.md5"
+
+  # Verify the tarball using md5sum or md5
+  if system("which md5sum > /dev/null")
+    matched = system("md5sum -c #{file}.md5")
+  elsif system("which md5 > /dev/null")
+    md5_out = %x[md5 -q #{file}].chomp
+    md5_actual = File.read("#{file}.md5").split[0]
+    matched = md5_out == md5_actual
+  else
+    puts "Cannot find md5sum or md5. Please install md5sum or md5 and try again"
+    exit 1
+  end
+
+  # Re-download tarball if corrupt; extract otherwise.
+  if !matched
+    sh "rm #{file} #{file}.md5"; download(url)
+  else
+    sh "tar xf #{file}"
+  end
 end
 
+# Connects to NCBI's FTP server, gets the URL of all database volumes and
+# returns them grouped by database name:
+#
+#     {'nr' => ['ftp://...', ...], 'nt' => [...], ...}
+#
 def databases
   host, dir = 'ftp.ncbi.nlm.nih.gov', 'blast/db'
   usr, pswd = 'anonymous', ENV['email']
@@ -20,10 +53,19 @@ def databases
   end
 end
 
+# Create user-facing task for each database to drive the download of its
+# volumes in parallel.
 databases.each do |name, files|
   multitask(name => files.map { |file| task(file) { download(file) } })
 end
 
+# Taxonomy database is different from sequence databases.
+task :taxdump do
+  download 'ftp://ftp.ncbi.nih.gov/pub/taxonomy/taxdump.tar.gz'
+end
+
+# List name of all databases that can be downloaded if executed without
+# any arguments.
 task :default do
-  puts databases.keys.join(', ')
+  puts databases.keys.push('taxdump').join(', ')
 end

data/ncbi-blast-dbs.gemspec

@@ -2,7 +2,7 @@ Gem::Specification.new do |s|
   s.authors     = ['Anurag Priyam']
   s.email       = ['anurag08priyam@gmail.com']
   s.name        = 'ncbi-blast-dbs'
-  s.version     = '0.0.2'
+  s.version     = '0.0.7'
   s.summary     = 'Fast download BLAST databases from NCBI.'
   s.description = <<DESC
 Downloads BLAST databases from NCBI. Database files (volumes) are downloaded in

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ncbi-blast-dbs
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.0.7
 platform: ruby
 authors:
 - Anurag Priyam
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-15 00:00:00.000000000 Z
+date: 2021-06-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -41,10 +41,12 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".ruby-version"
 - Gemfile
 - LICENSE.txt
 - README.md
 - bin/ncbi-blast-dbs
+- lib/http-ncbi-blast-dbs.rake
 - lib/ncbi-blast-dbs.rake
 - ncbi-blast-dbs.gemspec
 homepage: http://github.com/yeban/ncbi-blast-dbs
@@ -66,10 +68,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.2.5
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Fast download BLAST databases from NCBI.
 test_files: []
-has_rdoc: