ncbi-blast-dbs 0.0.6 → 0.0.7

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: bcce1b77f891ca2abb26a791de778fb9305d0e8a998f971eff97393e11f6ea87
+  data.tar.gz: 1dc5dc4d9f3d2bf0f6dea3b75bc04f1cefbece636598c941dcf08f9c2affba2a
+SHA512:
+  metadata.gz: fb6f3996b8e344a7e98af27c6187bc71a57cdf3996ed74d84243355c6c12aa9e122081b31b2b006d995fea04011c698cde49ef692c8b8eb6b45a790b2490d17f
+  data.tar.gz: bc9f835b65e8bb3fd7923d697a9b98096f2d5b5b8e9f7e5c0482f0996aece512397fecc70db78426e93bb91b99a199f7b9cd20bb48c0f46b915e5e580b7f7ef5

data/README.md

@@ -8,8 +8,8 @@ or re-downloaded if corrupt. Aborted downloads are safely resumed.
 
 `ncbi-blast-dbs` is faster than NCBI's `update_blastdb.pl`. But unlike
 `update_blastdb.pl`, which is a pure Perl script, `ncbi-blast-dbs` delegates
-download and checksum verification to `wget` and `md5sum` and is thus not as
-universal.
+download and checksum verification to `wget` and `md5sum` / `md5` and is thus
+not as universal.
 
 ### Installation
 

data/bin/ncbi-blast-dbs

@@ -1,13 +1,20 @@
 #!/usr/bin/env ruby
 
 require 'rake'
-import "#{File.dirname(__FILE__)}/../lib/ncbi-blast-dbs.rake"
 
 trap :INT do
   puts "Quitting ..."
   exit!
 end
 
-Rake.application.init 'ncbi-blast-dbs'
-Rake.application.load_imports
-Rake.application.top_level
+if ARGV.include? "http";
+  import "#{File.dirname(__FILE__)}/../lib/http-ncbi-blast-dbs.rake"
+  Rake.application.init 'http-ncbi-blast-dbs'
+  Rake.application.load_imports
+  Rake.application.top_level  
+else;
+  import "#{File.dirname(__FILE__)}/../lib/ncbi-blast-dbs.rake"
+  Rake.application.init 'ncbi-blast-dbs'
+  Rake.application.load_imports
+  Rake.application.top_level
+end

data/lib/http-ncbi-blast-dbs.rake

@@ -0,0 +1,84 @@
+require 'net/http'
+require 'uri'
+puts "using http-ncbi-dbs-dgs.rake"
+# Downloads tarball at the given URL if a local copy does not exist, or if the
+# local copy is older than at the given URL, or if the local copy is corrupt.
+def download(url, last_to_do)
+  file = File.basename(url)
+
+  # # Resume an interrupted download or fetch the file for the first time. If
+  # # the file on the server is newer, then it is downloaded from start.
+
+  sh "wget -Nc --no-verbose #{url}"
+  # If the local copy is already fully retrieved, then the previous command
+  # ignores the timestamp. So we check with the server again if the file on
+  # the server is newer and if so download the new copy.
+  sh "wget -N --no-verbose #{url}"
+  sh "wget -Nc --no-verbose #{url}.md5"
+  sh "wget -N --no-verbose #{url}.md5"
+  # Immediately download md5 and verify the tarball. Re-download tarball if
+  # corrupt; extract otherwise.
+  sh "md5sum -c #{file}.md5" do |matched, _|
+    if !matched
+      sh "rm #{file} #{file}.md5"; download(url)
+    # too many tar instances unzipping the same file clutter the system
+    elsif file == last_to_do;
+      sh "tar xfov #{file}"      
+    else
+      # at least nr and nt tarballs have identical files .?al; unsure of others
+      sh "tar xfov #{file} --exclude='*.?al' --exclude='taxdb*'"
+    end
+  end
+end
+
+
+def databases
+  method = 'https://'
+  host, dir = 'ftp.ncbi.nlm.nih.gov', 'blast/db'
+  uri = URI.parse(method + host + "/" + dir + "/")
+
+  response = Net::HTTP.get_response(uri)
+  body = response.body.split
+
+  array_of_files = []
+  body.each do |line|
+    # regex takes the raw http response, matches lines such as:
+    #    href="tsa_nt.06.tar.gz.md5">tsa_nt.06.tar.gz</a>
+    # Returns:
+    # tsa_nt.06.tar.gz
+    filenames_and_newlines = line[/(^href=".*">)(.*tar.gz|.*md5)(<\/a>)$/, 2]
+    array_of_files.append(filenames_and_newlines) unless filenames_and_newlines.nil?
+  end
+
+  # append the full path to file for downstream wget
+  array_of_files.map! { |string| "".concat("/blast/db/", string ) }
+  array_of_files.
+    map { |file| File.join(host, file) }.
+    select { |file| file.match(/\.tar\.gz$/) }.
+    group_by { |file| File.basename(file).split('.')[0] }
+end
+
+
+# Create user-facing task for each database to drive the download of its
+# volumes in parallel.
+databases.each do |name, files|
+  last = { name => files.last }
+  multitask(name => files.map { |file| task(file) { download(file, last.values.uniq) } })
+end
+
+# List name of all databases that can be downloaded if executed without
+# any arguments.
+task :default do
+  databases
+  puts databases.keys.push('taxdump').join(', ')
+end
+
+task :taxdump do
+  download('https://ftp.ncbi.nih.gov/pub/taxonomy/taxdump.tar.gz', "nil")
+end
+
+# Ruby being over my head, this is my quick-and-dirty way to trick it ignoring
+# "http" as a task rather than a specification. Happy for an expert to fix it up!
+task :http do
+  puts "using http method"
+end

data/lib/ncbi-blast-dbs.rake

@@ -6,20 +6,32 @@ def download(url)
   file = File.basename(url)
   # Resume an interrupted download or fetch the file for the first time. If
   # the file on the server is newer, then it is downloaded from start.
-  sh "wget -Nc #{url}"
+  sh "wget -Nc --no-verbose #{url}"
   # If the local copy is already fully retrieved, then the previous command
   # ignores the timestamp. So we check with the server again if the file on
   # the server is newer and if so download the new copy.
-  sh "wget -N #{url}"
-
-  # Immediately download md5 and verify the tarball. Re-download tarball if
-  # corrupt; extract otherwise.
-  sh "wget #{url}.md5 && md5sum -c #{file}.md5" do |matched, _|
-    if !matched
-      sh "rm #{file} #{file}.md5"; download(url)
-    else
-      sh "tar xvf #{file}"
-    end
+  sh "wget -N --no-verbose #{url}"
+
+  # Download Md5
+  sh "wget --no-verbose #{url}.md5"
+
+  # Verify the tarball using md5sum or md5
+  if system("which md5sum > /dev/null")
+    matched = system("md5sum -c #{file}.md5")
+  elsif system("which md5 > /dev/null")
+    md5_out = %x[md5 -q #{file}].chomp
+    md5_actual = File.read("#{file}.md5").split[0]
+    matched = md5_out == md5_actual
+  else
+    puts "Cannot find md5sum or md5. Please install md5sum or md5 and try again"
+    exit 1
+  end
+
+  # Re-download tarball if corrupt; extract otherwise.
+  if !matched
+    sh "rm #{file} #{file}.md5"; download(url)
+  else
+    sh "tar xf #{file}"
   end
 end
 

data/ncbi-blast-dbs.gemspec

@@ -2,7 +2,7 @@ Gem::Specification.new do |s|
   s.authors     = ['Anurag Priyam']
   s.email       = ['anurag08priyam@gmail.com']
   s.name        = 'ncbi-blast-dbs'
-  s.version     = '0.0.6'
+  s.version     = '0.0.7'
   s.summary     = 'Fast download BLAST databases from NCBI.'
   s.description = <<DESC
 Downloads BLAST databases from NCBI. Database files (volumes) are downloaded in

metadata

@@ -1,46 +1,39 @@
 --- !ruby/object:Gem::Specification
 name: ncbi-blast-dbs
 version: !ruby/object:Gem::Version
-  version: 0.0.6
-  prerelease: 
+  version: 0.0.7
 platform: ruby
 authors:
 - Anurag Priyam
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-04-30 00:00:00.000000000 Z
+date: 2021-06-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10.3'
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 10.3.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '10.3'
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 10.3.2
-description: ! 'Downloads BLAST databases from NCBI. Database files (volumes) are
-  downloaded in
-
+description: |
+  Downloads BLAST databases from NCBI. Database files (volumes) are downloaded in
   parallel; number of threads to use is determined automatically.  Database files
-
   are verified and extracted upon download.
-
-'
 email:
 - anurag08priyam@gmail.com
 executables:
@@ -48,36 +41,35 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
-- .ruby-version
+- ".ruby-version"
 - Gemfile
 - LICENSE.txt
 - README.md
 - bin/ncbi-blast-dbs
+- lib/http-ncbi-blast-dbs.rake
 - lib/ncbi-blast-dbs.rake
 - ncbi-blast-dbs.gemspec
 homepage: http://github.com/yeban/ncbi-blast-dbs
 licenses:
 - MIT
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 1.8.23.2
+rubygems_version: 3.0.3
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Fast download BLAST databases from NCBI.
 test_files: []