ncbi-taxonomy 0.2.7

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b77d9733ddf67d4f5ee94e8beae5d55826ba6c1e
+  data.tar.gz: c29cb5552e8e97c991ece58aa8eb0eb413a28a6b
+SHA512:
+  metadata.gz: 541dcb47a69210bd72a562027c8950d5bcc2e8e3bbeaeb6754469427599de0effb71769efd202d3c0d6c9104dcc3c2479be045cb916a130577241f928b819f6b
+  data.tar.gz: b87802c2886c9ad7c88fd70a87fe91af302111a1792acadc393347c5a298dbf9341a522bd470ec930a5af2627657202779dfaefebf8b5f3251d7c07961c0f191

data/bin/ncbi_taxonomy

@@ -0,0 +1,129 @@
+#!/usr/bin/env ruby
+require 'ncbi_taxonomy'
+require 'ncbi_taxonomy_update'
+
+def help msg
+	STDERR.puts msg
+	STDERR.puts "usage: #{$0.split("/")[-1]} <command> {-r} {-f} <args>"
+	STDERR.puts
+	STDERR.puts "These commands are used for search."
+	STDERR.puts "    name      Search taxonomic names using organism name with all ranks"
+	STDERR.puts "              example) $ ncbi_taxonomy name \"Escherichia coli SE11\""
+	#STDERR.puts "              example using file) ncbi_taxonomy name -f list.txt"
+	STDERR.puts
+	STDERR.puts "    fname     Search taxonomic names using organism name  with fixed ranks"
+	STDERR.puts "              Superkingdom(aka Domain) Phylum Class Order Family Genus Species strain"
+	STDERR.puts "              example) $ ncbi_taxonomy fname \"Acidibacter ferrireducens\""
+	#STDERR.puts "              example using file) ncbi_taxonomy fname -f list.txt"
+	STDERR.puts
+	STDERR.puts "    id        Search taxonomic names using NCBI taxon id with all ranks"
+	STDERR.puts "              example) $ ncbi_taxonomy id 409438"
+	#STDERR.puts "              example using file) ncbi_taxonomy id -f list.txt"
+	STDERR.puts
+	STDERR.puts "    fid       Search taxonomic names using NCBI taxon id with fixed ranks"
+	STDERR.puts "              example) $ ncbi_taxonomy fid 409438"
+	#STDERR.puts "              example using file) ncbi_taxonomy fid -f list.txt"
+	STDERR.puts
+	STDERR.puts "Common option for search command"
+	STDERR.puts "    -r        show each rank name in the result (default is hiding rank name)"
+	STDERR.puts "              example) $ ncbi_taxonomy name -r \"Escherichia coli SE11\""
+	STDERR.puts
+	STDERR.puts "We recommend to download the latest database using this command."
+	STDERR.puts "    update    Update the NCBI Taxonomy database"
+	STDERR.puts "              downloaded database is stored in $HOME/.ncbi_taxonomy/"
+	STDERR.puts "              example) $ ncbi_taxonomy update"
+	STDERR.puts
+	STDERR.puts "Common option for update command" 
+	STDERR.puts "    -f        force to update teh database"
+	STDERR.puts "              example) $ ncbi_taxonomy -f update"
+	#STDERR.puts "              example for forcing to download) ncbi_taxonomy update -f"
+	STDERR.puts
+	STDERR.puts "About rank type"
+	STDERR.puts "    all ranks   : all ranks registered in NCBI Taxonomy database"
+	STDERR.puts "    fixed ranks : major ranks for comman usage (especially microorganism)"
+	STDERR.puts "                  ==> Superkingdom(aka Domain) Phylum Class Order Family Genus Species strain\""
+	STDERR.puts
+	STDERR.puts "Bioinformatician can access methods of this software directly using Ruby."
+	STDERR.puts "More information is in GitLab web page."
+	STDERR.puts "https://gitlab.com/javamint/ncbi-taxonomy"
+	STDERR.puts 
+	exit 1
+end
+
+def search command, arg, opts
+	t = Taxonomy.new
+	out = nil
+	if command =~ /^na/
+		outs = t.get_allrank_by_name arg
+		STDERR.puts "[WARNING] name in multiple taxon" if outs.size > 1
+	elsif command =~ /^fn/
+		outs = t.get_fixedrank_by_name arg
+		STDERR.puts "[WARNING] name in multiple taxon" if outs.size > 1
+	elsif command =~ /^id/
+		outs = t.get_allrank_by_id arg.to_i
+		outs = [ outs ]
+	elsif command =~ /^fi/
+		outs = t.get_fixedrank_by_id arg.to_i
+		outs = [ outs ]
+	else
+		help "Incorrect command: #{command}"
+	end
+
+	if opts.index('-r') != nil 
+		outs.each do |out|
+			prt = Array.new
+			out.each do |x|
+				prt << "#{x[1]} (#{x[0]})" 
+			end
+			puts prt.join("\t")
+		end
+	else
+		outs.each do |out|
+			prt = Array.new
+			out.each do |x|
+				prt << x[1]
+			end
+			puts prt.join("\t")
+		end
+
+	end
+end
+
+def update force_flag
+	update = Update.new
+	
+	if update.status || force_flag
+		STDERR.print "NCBI Taxonomy database downloading "
+		update.do
+		STDERR.puts "DONE"
+	elsif update.status == false
+		STDERR.puts "Current local database is the latest one."
+	else
+		STDERR.puts update.status
+		exit 1
+	end
+end
+
+help '' if ARGV.size == 0
+
+args = Array.new
+opts = Array.new
+command = ARGV.shift
+case command
+when 'update'
+	ARGV.shift == '-f' ? flag = true  : flag = false
+	update flag
+else
+	while ARGV.size > 0
+		arg = ARGV.shift
+		if arg =~ /^-/
+			opts << arg
+		else
+			args << arg
+		end
+	end
+
+	args.each do |x|
+		search command, x, opts
+	end
+end

data/lib/ncbi_taxonomy.rb

@@ -0,0 +1,160 @@
+require 'sqlite3'
+
+class Taxonomy
+	def using_unique_name
+		{ "Ponticoccus" => 1, "Bacillus" => 1 }
+	end
+
+	def tax_rank_fixed
+		{ "superkingdom" => 0, "phylum" => 4, "class" => 7, "order" => 11, "family" => 16, "genus" => 21, "species" => 24, "strain" => 28 }
+	end
+
+	def tax_rank_all
+		{ "superkingdom" => 0, "kingdom" => 1, "subkingdom" => 2, "superphylum" => 3, "phylum" => 4, "subphylum" => 5, "superclass" => 6, "class" => 7, "infraclass" => 8, "subclass" => 9, "superorder" => 10, "order" => 11, "suborder" => 12, "infraorder" => 13, "parvorder" => 14, "superfamily" => 15, "family" => 16, "subfamily" => 17, "tribe" => 18, "subtribe" => 19, "genus" => 20, "subgenus" => 21, "species group" => 22, "species subgroup" => 23, "species" => 24, "subspecies" => 25, "varietas" => 26, "forma" => 27, "no rank" => 28 }
+	end
+
+	def initialize
+		@home_dir = Dir.home
+		@work_dir = @home_dir + "/.ncbi_taxonomy"
+		@taxdb_release = @work_dir + "/taxonomy.db"
+		begin
+			@db = SQLite3::Database.new @taxdb_release
+		rescue SQLite3::CantOpenException => e
+			STDERR.puts "Please download the NCBI Taxonomy database using 'ncbi_taxonomy update' command."
+			STDERR.puts "[MSG]" + e.message
+			exit 1
+		end
+	end
+
+	def memory
+		tmp = SQLite3::Database.new ':memory:'
+		backup = SQLite3::Backup.new tmp, 'main', @db, 'main'
+		backup.step -1
+		backup.finish
+		@db = tmp
+	end
+
+	def check_sqlite_version
+		rs = @db.execute "SELECT SQLITE_VERSION()"
+		Gem::Version.new(rs[0][0]) >= Gem::Version.new('3.8.3')
+	end
+
+	def get_taxonids_by_name name
+		name = SQLite3::Database.quote name
+		out = @db.execute "SELECT DISTINCT tax_id FROM names WHERE name_txt='#{name}'"
+		out.flatten
+	end
+
+	def get_names_by_taxonid id
+		@db.execute "SELECT name_class, name_txt FROM names WHERE tax_id=#{id}"
+	end
+
+	def get_scientific_name_by_names names
+		names.each {|x| return x[1] if x[0] == 'scientific name' }
+	end
+
+	def get_scientific_name_by_id id
+		id = id.to_i
+		out = @db.execute "SELECT name_txt FROM names WHERE tax_id=#{id} AND name_class='scientific name'"
+		out[0][0]
+	end
+	
+	def get_all_names_by_id id
+		id = id.to_i
+		out = @db.execute "SELECT name_txt FROM names WHERE tax_id=#{id}"
+		out
+	end
+
+	def get_rank_ptaxonid_by_id id
+		id = id.to_i
+		out = @db.execute "SELECT parent_tax_id, rank FROM nodes WHERE tax_id=#{id}"
+		out[0]
+	end
+
+	def get_rank_ptaxonid_scientificname_by_id id
+		id = id.to_i
+		out = @db.execute "SELECT nodes.parent_tax_id, nodes.rank, names.name_txt FROM nodes, names WHERE nodes.tax_id=#{id} AND names.tax_id=#{id} AND names.name_class='scientific name'"
+		out[0]
+	end
+
+	def get_allrank_by_id id
+		id = id.to_i
+		out = Array.new
+		if check_sqlite_version
+			rs = @db.execute "WITH RECURSIVE allrank(id, pid, rank, name) AS ( VALUES(0, #{id}, 'no_rank', 'Homo sapiens javamintus') UNION ALL SELECT nodes.tax_id, nodes.parent_tax_id, nodes.rank, names.name_txt FROM nodes, names, allrank WHERE nodes.tax_id=allrank.pid AND names.tax_id = nodes.tax_id AND names.name_class='scientific name' AND nodes.tax_id<>1) SELECT * FROM allrank;"
+			rs[1..-2].each {|x| out << [ x[2], x[3] ] }
+		else
+			while true
+				rs = self.get_rank_ptaxonid_scientificname_by_id id
+				out << [ rs[1], rs[2] ]
+				break if rs[1] == 'superkingdom'
+				id = rs[0]
+			end
+		end
+
+		out = self.get_allrank_by_id(self.get_missing_id id).reverse  if out.size == 0 && id > -1
+		out.reverse
+	end
+
+	def get_fixedrank_by_id id
+		id = id.to_i
+		ranks = self.get_allrank_by_id id
+		arr = Array.new
+		pos = 0	
+		alt_name = ''
+		ranks.each do |rank, name|
+			rank_fixed_no = tax_rank_fixed[rank]
+			rank_all_no = tax_rank_all[rank]
+			if rank_fixed_no != nil
+				this_rank_fixed = tax_rank_fixed.to_a.index [rank, rank_fixed_no]
+				if arr.size < this_rank_fixed
+					(arr.size...this_rank_fixed).each do |x|
+						arr << [ tax_rank_fixed.to_a[x][0], "@#{alt_name}_#{tax_rank_fixed.to_a[x][0]}" ]
+						pos += 1
+					end
+				end
+				arr << [ rank, name ]
+				pos += 1
+				alt_name = name
+			elsif arr.size == 7 && rank_all_no == 28
+				arr << [ 'strain', name ]
+			elsif rank_all_no != 28
+				alt_name = name if tax_rank_fixed.to_a[pos-1][1] > tax_rank_all[rank]
+			end
+		end
+		if arr.size < 7 && arr.size >= 0
+			(arr.size..7).each do |x|
+				arr << [ tax_rank_fixed.to_a[x][0], nil ]
+			end
+		elsif arr.size == 7
+			arr << [ 'strain', arr[-1][1] ]
+		end
+		arr
+	end
+
+	def get_allrank_by_name name
+		out = Array.new
+		self.get_taxonids_by_name(name).each {|x| out << self.get_allrank_by_id(x) }
+		out
+	end
+
+	def get_fixedrank_by_name name
+		out = Array.new
+		self.get_taxonids_by_name(name).each {|x| out << self.get_fixedrank_by_id(x) }
+		out
+	end
+
+	def get_missing_id id
+		rs = @db.execute "SELECT tax_id FROM delnodes WHERE tax_id='#{id}'"
+		if rs.size > 0
+			return -1
+		else
+			rs = @db.execute "SELECT new_tax_id FROM merged WHERE old_tax_id=#{id}"
+			if rs.size == 1
+				return rs[0][0].to_i
+			else
+				return -1
+			end
+		end
+	end
+end

data/lib/ncbi_taxonomy_update.rb

@@ -0,0 +1,179 @@
+require 'sqlite3'
+require 'fileutils'
+# Bug : 2014. 04. 05
+#	Error: citations.dmp.trim line 32659: expected 7 columns of data but found 8
+# 	Actually this software does not use this table. Therefore, this error can be ignored.
+# 	Later, I should fix this.
+
+class Update	
+	def initialize
+		@home_dir = Dir.home
+		@work_dir = @home_dir + "/.ncbi_taxonomy"
+		md5_file = @work_dir + "/taxdump.tar.gz.md5"
+		md5_old_file = @work_dir + "/taxdump.tar.gz.md5.old"
+		@taxdb = @work_dir + "/taxonomy.db.prep"
+		@taxdb_release = @work_dir + "/taxonomy.db"
+		@status = nil
+
+		# check workinng directory, if not exist, make it.
+		if File.exist?(@work_dir)
+			if !File.directory?(@work_dir)
+				@status = "This software uses $HOME/.ncbi_taxonomy directory. However, in your home directory there is same name of file. We recommend you change that file name to another name."
+				return
+			else
+				Dir.chdir @work_dir
+				if File.exist?(md5_file)
+					`rm -f #{md5_old_file}`
+					File.rename(md5_file, md5_old_file)
+					self.download_md5
+					`diff #{md5_file} #{md5_old_file}`
+					if $?.exitstatus == 0
+						@status = false
+						return
+					else
+						@status = true
+						return
+					end
+				else
+					self.download_md5
+				end
+			end
+		else
+			Dir.mkdir @work_dir
+			Dir.chdir @work_dir
+			self.download_md5
+		end
+
+		@status = true
+		return
+	end
+
+	def status
+		return @status
+	end
+
+	def download_md5
+		`curl -s ftp://ftp.ncbi.nih.gov/pub/taxonomy/taxdump.tar.gz.md5 > taxdump.tar.gz.md5 2>/dev/null`
+	end
+	
+	def download_dump
+		`curl -s ftp://ftp.ncbi.nih.gov/pub/taxonomy/taxdump.tar.gz 2>/dev/null | tar zxf - `
+	end
+	
+	# substitute some characters
+	def substitution
+		Dir.entries(@work_dir).each do |file|
+			if file =~ /dmp$/
+				#STDERR.puts "treating #{file}"
+				File.open(@work_dir+"/"+file+".trim","w") do |out|
+					out << File.open(@work_dir+"/"+file).read.force_encoding('iso-8859-1').encode('utf-8').gsub(/([^|])	([^|])/,'\1 \2').gsub(/\t\|\t/,"\t").gsub(/\t\|$/,"").gsub(/\"/,"%22")
+				end
+			end
+		end
+	end
+
+	def release
+		begin
+			FileUtils.rm "#{@taxdb_release}.old"
+		rescue Errno::ENOENT => e
+		end
+		begin
+			File.rename(@taxdb_release, @taxdb_release+".old")
+		rescue Errno::ENOENT => e
+		end
+		File.rename(@taxdb, @taxdb_release)
+	end
+
+	def do
+		self.download_dump
+		self.substitution
+		self.load_db
+		self.release
+	end
+	
+	def load_db
+		sql = <<EOF
+PRAGMA page_size=4096;
+PRAGMA main.locking_mode=EXCLUSIVE;
+
+.separator '\t'
+
+CREATE TABLE citations (
+cit_id BIGINT,
+cit_key VARCHAR(255),
+pubmed_id BIGINT,
+medline_id BIGINT,
+ur LONGTEXT,
+text LONGTEXT,
+taxid_list LONGTEXT
+);
+
+CREATE TABLE delnodes (
+tax_id BIGINT
+);
+
+CREATE TABLE division (
+division_id     BIGINT,
+division_cde VARCHAR(255),
+division_name VARCHAR(255),
+comments VARCHAR(255)
+);
+
+CREATE TABLE gencode (
+genetic_code_id INT,
+abbreviation VARCHAR(255),
+name VARCHAR(255),
+cde LONGTEXT,
+starts LONGTEXT
+);
+
+CREATE TABLE merged (
+old_tax_id BIGINT,
+new_tax_id BIGINT
+);
+
+CREATE TABLE names (
+tax_id BIGINT,
+name_txt VARCHAR(255),
+unique_name VARCHAR(255),
+name_class VARCHAR(255)
+);
+
+CREATE TABLE nodes (
+tax_id BIGINT,
+parent_tax_id BIGINT,
+rank VARCHAR(64), 
+embl_code VARCHAR(64),
+division_id INTEGER,
+inherited_div_flag BOOLEAN,
+genetic_code_id INTEGER,
+inherited_GC_flag BOOLEAN,
+mitochondrial_genetic_code_id INTEGER,
+inherited_MGC_flag BOOLEAN,
+GenBank_hidden_flag BOOLEAN,
+hidden_subtree_root_flag BOOLEAN,
+comments VARCHAR(255)
+);
+
+
+CREATE INDEX citations_idx ON citations(cit_id,cit_key,pubmed_id,medline_id,ur,text,taxid_list);
+CREATE INDEX delnodes_idx ON delnodes(tax_id);
+CREATE INDEX division_idx ON division(division_id,division_cde,division_name,comments);
+CREATE INDEX gencode_idx ON gencode(genetic_code_id,abbreviation,name,cde,starts);
+CREATE INDEX merged_idx ON merged(old_tax_id,new_tax_id);
+CREATE INDEX names_idx ON names(tax_id,name_txt,unique_name,name_class);
+CREATE INDEX nodes_idx ON nodes(tax_id,parent_tax_id,rank,embl_code,division_id,inherited_div_flag,genetic_code_id,inherited_GC_flag,mitochondrial_genetic_code_id,inherited_MGC_flag,GenBank_hidden_flag,hidden_subtree_root_flag,comments);
+
+
+.import citations.dmp.trim citations
+.import delnodes.dmp.trim delnodes
+.import division.dmp.trim division
+.import gencode.dmp.trim gencode
+.import merged.dmp.trim merged
+.import names.dmp.trim names
+.import nodes.dmp.trim nodes
+		
+EOF
+		`echo "#{sql}" | sqlite3 #{@taxdb} < /dev/stdin`
+	end
+end

metadata

@@ -0,0 +1,68 @@
+--- !ruby/object:Gem::Specification
+name: ncbi-taxonomy
+version: !ruby/object:Gem::Version
+  version: 0.2.7
+platform: ruby
+authors:
+- Seok-Won Kim
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-08-15 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
+description: This gem supports essential functions for fast access of NCBI Taxonomy
+  database
+email: javamint@gmail.com
+executables:
+- ncbi_taxonomy
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/ncbi_taxonomy
+- lib/ncbi_taxonomy.rb
+- lib/ncbi_taxonomy_update.rb
+homepage: https://gitlab.com/javamint/ncbi-taxonomy
+licenses:
+- Nonstandard
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: NCBI Taxonomy search using local repository
+test_files: []