semtools 0.1.2 → 0.1.8

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8fc29918a31045893647355dd72264a04386c5171c48ea868f7e9bbc93062151
-  data.tar.gz: 692ce02343cb00ac37bbc34476da08386bedf0eaca7946689eb62c9a1f06d555
+  metadata.gz: a3f63cc6548a9938e31121d2018d1c1c477987007c5d253b5fa814a285bdb576
+  data.tar.gz: e1911d3157c3046590ca13bc86215d2260b4a8b2b1b25affa5c2673881036795
 SHA512:
-  metadata.gz: 1b52667c81a0a25786b91156e9ed88a8de47e86fd18baddffc43b05ff199f95129b09da4e03025b6fb709d18a0274e22bf4a55c81471fda748e75aadca4d6ef1
-  data.tar.gz: 46e5b49f611c021ee8576a522a0a6ef22a8b9ed349084dadb9e44fd76c712c05221e6314985f08bdba575ac2dd849f1f14d84d5ae686889f33fac993132a8372
+  metadata.gz: 30c95df80957a4a35b6fea05b9552352f529d8e45c10f6b128924a3ce2ee5d90e92a1e9d5fe0016d25538147e12d3a9199c81222642c94cdd0eb3c89eea168ef
+  data.tar.gz: ddc9e600fd984e68d060b7be05adf27b3f20bb67e638d42acc4b9b156eedabfce20d6f588a03d1fbc2948fedbd80d498f1767c0e3f8ea03720fa0ca327b95f3c

data/Gemfile

@@ -5,5 +5,8 @@ git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
 # Specify your gem's dependencies in semtools.gemspec
 gemspec
 
-gem "rake", "~> 12.0"
+gem "rake", "~> 13.0"
 gem "minitest", "~> 5.0"
+
+expcalc_dev_path = File.expand_path('~/dev_gems/expcalc')
+gem "expcalc", github: "seoanezonjic/expcalc", branch: "master" if Dir.exist?(expcalc_dev_path)

data/bin/onto2json.rb

@@ -18,14 +18,20 @@ OptionParser.new do |opts|
   opts.banner = "Usage: #{__FILE__} [options]"
 
   options[:input_file] = nil
-  opts.on("-i", "--input_file PATH", "Input file with ontology in OBO format") do |data|
+  opts.on("-i", "--input_file FILE", "Input file with ontology in OBO format") do |data|
     options[:input_file] = data
   end
 
   options[:output_file] = nil
-  opts.on("-o", "--output_file PATH", "Output path") do |data|
+  opts.on("-o", "--output_file FILE", "Output path") do |data|
     options[:output_file] = data
   end
+
+  options[:build] = false
+  opts.on("-b", "--build", "Activate build mode (calculate dictionaries)") do
+    options[:build] = true
+  end
+
   
   opts.on_tail("-h", "--help", "Show this message") do
     puts opts
@@ -39,7 +45,7 @@ end.parse!
 # MAIN
 ##########################
 puts "Loading ontology ..."
-onto = Ontology.new(file: options[:input_file], load_file: true)
+onto = Ontology.new(file: options[:input_file], load_file: true, build: options[:build])
 puts "Exporting ontology to JSON ..."
 onto.write(options[:output_file])
 puts "Ontology exported"

data/bin/semtools.rb

@@ -0,0 +1,446 @@
+#! /usr/bin/env ruby
+ROOT_PATH = File.dirname(__FILE__)
+$LOAD_PATH.unshift(File.expand_path(File.join(ROOT_PATH, '..', 'lib')))
+EXTERNAL_DATA = File.expand_path(File.join(ROOT_PATH, '..', 'external_data'))
+
+require 'optparse'
+require 'down'
+require 'semtools'
+
+######################################################################################
+## METHODS
+######################################################################################
+def load_tabular_file(file)
+  records = []
+  File.open(file).each do |line|
+    line.chomp!
+    fields = line.split("\t")
+    records << fields
+  end
+  return records
+end
+
+def store_profiles(file, ontology)
+  file.each do |id, terms|
+    ontology.add_profile(id, terms)
+  end  
+end
+
+def load_value(hash_to_load, key, value, unique = true)
+   	query = hash_to_load[key]
+    if query.nil?
+       value = [value] if value.class != Array
+       hash_to_load[key] = value
+    else
+        if value.class == Array
+            query.concat(value)
+        else
+            query << value
+        end
+        query.uniq! unless unique == nil
+    end
+end
+
+def translate(ontology, type, options, profiles = nil)
+  not_translated = {}
+  if type == 'names'
+    ontology.profiles.each do |id, terms|
+      translation, untranslated = ontology.translate_ids(terms)
+      ontology.profiles[id] = translation  
+      not_translated[id] = untranslated unless untranslated.empty?
+    end  
+  elsif type == 'codes'
+    profiles.each do |id,terms|
+      translation, untranslated = ontology.translate_names(terms)
+      profiles[id] = translation
+      profiles[id] = profiles[id].join("#{options[:separator]}") 
+      not_translated[id] = untranslated unless untranslated.empty?
+    end    
+  end
+  if !not_translated.empty?
+    File.open(options[:untranslated_path], 'w') do |file|
+      not_translated.each do |id, terms|
+          file.puts([id, terms.join(";")].join("\t"))
+      end
+    end
+  end    
+end  
+
+def clean_profile(profile, ontology, options)
+	cleaned_profile = ontology.clean_profile_hard(profile)	
+	unless options[:term_filter].nil?
+		cleaned_profile.select! {|term| ontology.get_ancestors(term).include?(options[:term_filter])}
+	end	
+	return cleaned_profile
+end
+
+def clean_profiles(profiles, ontology, options)
+	removed_profiles = []
+	profiles.each do |id, terms|
+		cleaned_profile = clean_profile(terms, ontology, options)
+		profiles[id] = cleaned_profile
+		removed_profiles << id if cleaned_profile.empty?
+	end
+	removed_profiles.each{|rp| profiles.delete(rp)}
+	return removed_profiles
+end
+
+def expand_profiles(profiles, ontology, unwanted_terms = [])
+	profiles.each do |disease_id, terms|
+		terms.each do |term|
+	    	profiles[disease_id] << ontology.get_ancestors(term).difference(unwanted_terms)
+	  end
+	end
+end	
+
+def write_similarity_profile_list(input, onto_obj, similarity_type)
+  similarity_file = File.basename(input, ".*")+'_semantic_similarity_list'  
+  File.open(similarity_file, 'w') do |file|
+    onto_obj.profiles.each do |profile_query_key, profile_query_value|
+      onto_obj.profiles.each do |profile_search_key, profile_search_value|
+        file.puts([profile_query_key, profile_search_key, onto_obj.compare(profile_query_value, profile_search_value, sim_type: similarity_type)].join("\t"))
+      end
+    end 
+  end
+end
+
+def download(source, key, output)
+  source_list = load_tabular_file(source).to_h
+  external_data = File.dirname(source)
+  if key == 'list'
+    Dir.glob(File.join(external_data,'*.obo')){|f| puts f}
+  else
+    url = source_list[key]
+    if !output.nil?
+      output_path = output
+    else
+      file_name = key + '.obo'
+      if File.writable?(external_data)
+        output_path = File.join(external_data, file_name)
+      else
+        output_path = file_name
+      end
+    end
+    if !url.nil?
+      Down::NetHttp.download(url, destination: output_path, max_redirects: 5)
+      File.chmod(0644, output_path) # Correct file permissions set by down gem
+    end
+  end
+end
+
+def get_ontology_file(path, source)
+  if !File.exists?(path)
+    ont_index = load_tabular_file(source).to_h
+    if !ont_index[path].nil?
+      path = File.join(File.dirname(source), path + '.obo')
+    else
+      abort("Input ontology file not exists")
+    end
+  end
+  return path
+end
+
+def get_stats(stats)
+  report_stats = []
+  report_stats << ['Elements', stats[:count]]
+  report_stats << ['Elements Non Zero', stats[:countNonZero]]
+  report_stats << ['Non Zero Density', stats[:countNonZero].fdiv(stats[:count])]
+  report_stats << ['Max', stats[:max]]
+  report_stats << ['Min', stats[:min]]
+  report_stats << ['Average', stats[:average]]
+  report_stats << ['Variance', stats[:variance]]
+  report_stats << ['Standard Deviation', stats[:standardDeviation]]
+  report_stats << ['Q1', stats[:q1]]
+  report_stats << ['Median', stats[:median]]
+  report_stats << ['Q3', stats[:q3]]
+  return report_stats
+end
+
+
+
+
+
+
+####################################################################################
+## OPTPARSE
+####################################################################################
+options = {}
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{File.basename(__FILE__)} [options]"
+
+  options[:download] = nil
+  opts.on("-d", "--download STRING", "Download obo file from official resource. MONDO, GO and HPO are possible values.") do |item|
+    options[:download] = item
+  end
+
+  options[:input_file] = nil
+  opts.on("-i", "--input_file PATH", "Filepath of profile data") do |item|
+    options[:input_file] = item
+  end
+
+  options[:output_file] = nil
+  opts.on("-o", "--output_file PATH", "Output filepath") do |item|
+    options[:output_file] = item
+  end
+
+  options[:IC] = false
+  opts.on("-I", "--IC", "Get IC") do
+    options[:IC] = true
+  end
+
+  options[:ontology_file] = nil
+  opts.on("-O PATH", "--ontology_file PATH", "Path to ontology file") do |item|
+  	options[:ontology_file] = item
+  end
+
+  options[:term_filter] = nil
+  opts.on("-T STRING", "--term_filter STRING", "If specified, only terms that are descendants of the specified term will be kept on a profile when cleaned") do |item|
+  	options[:term_filter] = item.to_sym
+  end
+
+  options[:translate] = nil
+  opts.on("-t STRING", "--translate STRING", "Translate to 'names' or to 'codes'") do |item|
+  	options[:translate] = item
+  end
+  
+  opts.on("-s method", "--similarity method", "Calculate similarity between profile IDs computed by 'resnik', 'lin' or 'jiang_conrath' methods. ") do |sim_method|
+  	options[:similarity] = sim_method.to_sym
+  end
+
+  options[:clean_profiles] = false
+	opts.on("-c", "--clean_profiles", "Removes ancestors, descendants and obsolete terms from profiles") do
+  	options[:clean_profiles] = true
+  end
+
+  options[:removed_path] = 'rejected_profs'
+  opts.on("-r PATH", "--removed_path PATH", "Desired path to write removed profiles file") do |item|
+  	options[:removed_path] = item
+  end
+
+  options[:untranslated_path] = nil
+  opts.on("-u PATH", "--untranslated_path PATH", "Desired path to write untranslated terms file") do |item|
+    options[:untranslated_path] = item
+  end
+
+  options[:keyword] = nil
+  opts.on("-k STRING", "--keyword STRING", "regex used to get xref terms in the ontology file") do |item|
+    options[:keyword] = item
+  end
+
+  options[:xref_sense] = :byValue
+  opts.on("--xref_sense ", "Ontology-xref or xref-ontology. By default xref-ontology if set, ontology-xref") do
+    options[:xref_sense] = :byTerm
+  end
+
+  options[:expand_profiles] = false
+  opts.on("-e", "--expand_profiles", "Expand profiles adding ancestors") do
+    options[:expand_profiles] = true
+  end
+
+  options[:unwanted_terms] = []
+  opts.on("-U", "--unwanted_terms STRING", "Comma separated terms not wanted to be included in profile expansion") do |item|
+    options[:unwanted_terms] = item
+  end
+
+  options[:separator] = ";"
+  opts.on("-S STRING", "--separator STRING", "Separator used for the terms profile") do |sep|
+    options[:separator] = sep
+  end     
+
+  options[:childs] = [[], '']
+  opts.on("-C STRING", "--childs STRING", "Term code list (comma separated) to generate child list") do |item|
+    if item.include?('/')
+      modifiers, terms = item.split('/')
+    else
+      modifiers = ''
+      terms = item
+    end
+    terms = terms.split(',').map{|t| t.to_sym}
+    options[:childs] = [terms, modifiers]
+  end   
+
+  options[:statistics] = false
+  opts.on("-n", "--statistics", "To obtain main statistical descriptors of the profiles file") do
+    options[:statistics] = true
+  end
+
+  options[:list_translate] = nil
+  opts.on("-l STRING", "--list_translate STRING", "Translate to 'names' or to 'codes' input list") do |sep|
+    options[:list_translate] = sep
+  end     
+
+  options[:subject_column] = 0
+  opts.on("-f NUM", "--subject_column NUM", "The number of the column for the subject id") do |ncol|
+    options[:subject_column] = ncol.to_i
+  end
+
+  options[:annotations_column] = 1
+  opts.on("-a NUM", "--annotations_column NUM", "The number of the column for the annotation ids") do |ncol|
+    options[:annotations_column] = ncol.to_i
+  end
+
+
+  options[:list_term_attributes] = false
+  opts.on("--list_term_attributes", "The number of the column for the annotation ids") do
+    options[:list_term_attributes] = true
+  end
+
+end.parse!
+
+####################################################################################
+## MAIN
+####################################################################################
+ont_index_file = File.join(EXTERNAL_DATA, 'ontologies.txt')
+if !options[:download].nil?
+  download(ont_index_file, options[:download], options[:output_file])
+  Process.exit
+end
+
+if !options[:ontology_file].nil?
+  options[:ontology_file] = get_ontology_file(options[:ontology_file], ont_index_file)
+end
+ontology = Ontology.new(file: options[:ontology_file], load_file: true)
+
+if !options[:input_file].nil?
+  data = load_tabular_file(options[:input_file])
+  if options[:list_translate].nil? || !options[:keyword].nil?
+    data.map!{|row| 
+      [row[options[:subject_column]], 
+       row[options[:annotations_column]].split(options[:separator]).map!{|term| term.to_sym}]
+    }
+    store_profiles(data, ontology) if options[:translate] != 'codes' && options[:keyword].nil?
+  end
+end
+
+if !options[:list_translate].nil?
+  data.each do |term|
+    if options[:list_translate] == 'names'
+      translation, untranslated = ontology.translate_ids(term)
+    elsif options[:list_translate] == 'codes'
+      translation, untranslated = ontology.translate_names(term)
+    end
+    puts "#{term.first}\t#{translation.empty? ? '-' : translation.first}"
+  end
+  Process.exit
+end
+
+if options[:translate] == 'codes'
+  profiles = {}
+  data.each do |id, terms|
+    load_value(profiles, id, terms)
+    profiles[id] = terms.split(options[:separator])
+  end
+  translate(ontology, 'codes', options, profiles)
+  store_profiles(profiles, ontology)
+end
+   
+if options[:clean_profiles]
+	removed_profiles = clean_profiles(ontology.profiles, ontology, options)	
+	if !removed_profiles.nil? && !removed_profiles.empty?
+      File.open(options[:removed_path], 'w') do |f|
+          removed_profiles.each do |profile|
+              f.puts profile
+          end
+      end
+	end
+end
+
+if options[:expand_profiles]
+  expanded_profiles = expand_profiles(ontology.profiles, ontology, options[:unwanted_terms])
+end 
+
+if !options[:similarity].nil?
+  write_similarity_profile_list(input = options[:input_file], onto_obj=ontology, similarity_type = options[:similarity])
+end 
+
+
+if options[:IC]
+  ontology.add_observed_terms_from_profiles
+  by_ontology, by_freq = ontology.get_profiles_resnik_dual_ICs
+  ic_file = File.basename(options[:input_file], ".*")+'_IC_onto_freq'
+  File.open(ic_file , 'w') do |file|
+    ontology.profiles.keys.each do |id|
+        file.puts([id, by_ontology[id], by_freq[id]].join("\t"))
+    end       
+  end
+end    
+
+if options[:translate] == 'names'
+  translate(ontology, 'names', options)  
+end
+
+if !options[:childs].first.empty?
+  terms, modifiers = options[:childs]
+  all_childs = []
+  terms.each do |term|
+   childs = ontology.get_descendants(term)
+   all_childs = all_childs | childs
+  end
+  if modifiers.include?('r')
+    relations = []
+    all_childs = all_childs | terms # Add parents that generated child list
+    all_childs.each do |term|
+      descendants = ontology.get_direct_descendants(term)
+      if !descendants.nil?
+        descendants.each do |desc|
+          relations << [term, desc]
+        end
+      end
+    end
+    relations.each do |rel| 
+      rel, _ = ontology.translate_ids(rel) if modifiers.include?('n')
+      puts rel.join("\t")
+    end
+  else
+    all_childs.each do |c| 
+      if modifiers.include?('n') 
+        puts ontology.translate_id(c) 
+      else
+        puts c
+      end
+    end
+  end
+end
+
+if !options[:output_file].nil?
+  File.open(options[:output_file], 'w') do |file|
+    ontology.profiles.each do |id, terms|
+      file.puts([id, terms.join("|")].join("\t"))
+    end
+  end         
+end
+
+if options[:statistics]
+  get_stats(ontology.profile_stats).each do |stat|
+    puts stat.join("\t")
+  end
+end
+
+if options[:list_term_attributes]
+  term_attributes = ontology.list_term_attributes
+  term_attributes.each do |t_attr|
+    t_attr[0] = t_attr[0].to_s
+    puts t_attr.join("\t")
+  end
+end
+
+if !options[:keyword].nil?
+  xref_translated = []
+  ontology.calc_dictionary(:xref, select_regex: /(#{options[:keyword]})/, store_tag: :tag, multiterm: true, substitute_alternatives: false)
+  dict = ontology.dicts[:tag][options[:xref_sense]]
+  data.each do |id, prof|
+    xrefs = []
+    prof.each do |t|
+      query = dict[t.to_s]
+      xrefs.concat(query) if !query.nil?
+    end
+    xref_translated << [id, xrefs] if !xrefs.empty?
+  end
+  File.open(options[:output_file], 'w') do |f|
+    xref_translated.each do |id, prof|
+      prof.each do |t|
+        f.puts [id, t].join("\t")
+      end
+    end
+  end
+end

data/bin/strsimnet.rb

@@ -111,12 +111,11 @@ texts2compare = load_table_file(input_file = options[:input_file],
                                  targetCol = options[:cindex],
                                  filterCol = options[:findex],
                                  filterValue = options[:filter_value])
-
 # Verbose point
 puts "Calculating similitude for (" + texts2compare.length.to_s + ") elements"
 
 # Obtain all Vs all
-similitudes_AllVsAll = similitude_network(texts2compare,options[:rm_char])
+similitudes_AllVsAll = similitude_network(texts2compare, charsToRemove: options[:rm_char])
 
 # Verbose point
 puts "Writing output file ..."

data/external_data/ontologies.txt

@@ -0,0 +1,4 @@
+GO	http://purl.obolibrary.org/obo/go/go-basic.obo
+HPO	http://purl.obolibrary.org/obo/hp.obo
+MONDO	http://purl.obolibrary.org/obo/mondo.obo
+EFO	http://www.ebi.ac.uk/efo/efo.obo