pets 0.2.3 → 0.2.4

data/lib/pets/genomic_features.rb

@@ -0,0 +1,144 @@
+class Genomic_Feature
+	#If any method use gen_fet as name is a Genomic_Feature object
+	def initialize(feat_array) # [[chr1, start1, stop1],[chr1, start1, stop1]]
+		@regions = {}
+		@reg_id = -1
+		load_features(feat_array)
+	end
+
+	def load_features(feat_array)
+		feat_array.each do |chr, start, stop|
+			chr = chr.to_sym
+			region = {start: start, stop: stop, to: @reg_id +=1 }
+			add_record(@regions, chr, region)
+		end
+	end
+
+	def length
+		return @regions.length
+	end
+
+	def each()
+		@regions.each do |chr, regs|
+			regs.each do |region|
+				yield(chr, region)
+			end
+		end
+	end
+
+	def get_chr
+		return @regions.keys
+	end
+
+	def get_sizes
+		sizes = []
+		each do |chr, region|
+			size = region[:stop] - region[:start] + 1
+			sizes << size
+		end
+		return sizes
+	end
+
+	def get_summary_sizes
+		sizes = Hash.new(0)
+		each do |chr, region|
+			size = region[:stop] - region[:start] + 1
+			sizes[size] += 1
+		end
+		return sizes.to_a.sort!{|s| s[1] <=> s[1] }
+	end
+
+	def merge(gen_fet, to = nil) # 'to' the regions must be connected "to" given id
+		gen_fet.each do |chr, region|
+			to.nil? ? region[:to] = @reg_id +=1 : region[:to] = to # handle id custom or default
+			add_record(@regions, chr, region)
+		end
+	end
+
+	def get_reference_overlaps(genomic_ranges, reference)
+		overlaps = []
+		reference.each do |start, stop|
+			reg_ids = []
+			genomic_ranges.each do |reg|
+				reg_ids << reg[:to] if coor_overlap?(start, stop, reg)
+			end
+			overlaps << reg_ids.uniq
+		end
+		return overlaps
+	end
+
+	def generate_cluster_regions(meth, tag, ids_per_reg = 1)
+		compute_windows(meth) # Get putative genome windows
+		patients_out_of_cluster = 0
+		ids_by_cluster = {}
+		annotated_full_ref = [] # All reference windows wit uniq id and chr tagged
+		@regions.each do |chr, regs|
+			reference = @windows[chr]
+			overlaps = get_reference_overlaps(regs, reference) # See what patient has match with a overlap region
+			clust_numb = 0
+			reference.each_with_index do |ref, i|
+				current_ids = overlaps[i]
+				if current_ids.length > ids_per_reg
+					clust_id = "#{chr}.#{clust_numb +=1}.#{tag}.#{current_ids.length}"
+					current_ids.each do |curr_id|
+						add_record(ids_by_cluster, curr_id, clust_id, true)
+					end
+					annotated_full_ref << ref.dup.concat([chr, clust_id])
+				end
+			end
+		end
+		return ids_by_cluster, annotated_full_ref
+	end
+
+	def compute_windows(meth)
+		@windows = {}
+		@regions.each do |chr, regs|
+			chr_windows = nil
+			if meth == :reg_overlap
+				chr_windows = compute_region_overlap_windows(regs)
+			end
+			@windows[chr] = chr_windows
+		end
+	end
+
+	private
+	
+	def add_record(hash, key, record, uniq=false)
+		query = hash[key]
+		if query.nil?
+			hash[key] = [record]
+		elsif !uniq # We not take care by repeated entries
+			query << record
+		elsif !query.include?(record) # We want uniq entries
+			query << record
+		end
+	end
+
+	def compute_region_overlap_windows(genomic_ranges)
+		reference = []
+		reference.concat(genomic_ranges.map{|gr| gr[:start]})# get start
+		reference.concat(genomic_ranges.map{|gr| gr[:stop]})# get stop
+		reference.uniq!
+		reference.sort!
+		#Define overlap ranges
+		final_reference = []
+		reference.each_with_index do |coord,i|
+			next_coord = reference[i + 1]
+			final_reference << [coord, next_coord] if !next_coord.nil? 
+		end
+		return final_reference
+	end
+
+	def coor_overlap?(start, stop, reg)
+		overlap = false
+		reg_start = reg[:start]
+		reg_stop = reg[:stop]
+		if (start <= reg_start && stop >= reg_stop) ||
+			(start > reg_start && stop < reg_stop) ||
+			(stop > reg_start && stop <= reg_stop) ||
+			(start >= reg_start && start < reg_stop)
+			overlap = true
+		end
+		return overlap
+	end
+end

data/lib/pets/io.rb

@@ -0,0 +1,457 @@
+require 'csv'
+
+def load_hpo_ontology(hpo_file, excluded_hpo_file)
+  hpo = nil
+  if !hpo_file.include?('.json')
+    if !excluded_hpo_file.nil?
+      hpo = Ontology.new(file: hpo_file, load_file: true, removable_terms: read_excluded_hpo_file(excluded_hpo_file))
+    else
+      hpo = Ontology.new(file: hpo_file, load_file: true)
+    end
+  else
+    hpo = Ontology.new
+    hpo.read(hpo_file)
+    if !excluded_hpo_file.nil?
+      hpo.add_removable_terms(read_excluded_hpo_file(excluded_hpo_file))
+      hpo.remove_removable()
+      hpo.build_index()
+    end
+  end
+  return hpo
+end
+
+def read_excluded_hpo_file(file)
+  excluded_hpo = []
+  File.open(file).each do |line|
+    excluded_hpo << line.chomp
+  end
+  return excluded_hpo
+end
+
+def write_hash(hash, file_path, header = [])
+  File.open(file_path, 'w') do |handler|
+    handler.puts header.join("\t") if !header.empty?
+    hash.each do |key, array|
+      handler.puts "#{key}\t#{array.join("\t")}"
+    end
+  end
+end
+
+def write_array(array, file_path)
+  File.open(file_path, 'w') do |handler|
+    array.each do |record|
+      if record.class == String
+        line = record
+      else
+        line = record.join("\t")
+      end
+      handler.puts line  
+    end
+  end
+end
+
+def write_matrix_for_R(matrix, x_names, y_names, file)
+  File.open(file, 'w') do |f|
+    f.puts x_names.join("\t")
+    matrix.each_with_index do |row, i|
+      f.puts [y_names[i]].concat(row).join("\t")
+    end
+  end
+end
+
+def write_cluster_ic_data(all_ics, profile_lengths, cluster_ic_data_file, limit)
+  File.open(cluster_ic_data_file, 'w') do |f|
+    f.puts %w[cluster_id ic Plen].join("\t")
+    all_ics.each_with_index do |cluster_ics, i|
+      break if i == limit
+      cluster_length = cluster_ics.length
+      cluster_ics.each_with_index do |clust_ic, j|
+        f.puts "#{cluster_length}_#{i}\t#{clust_ic}\t#{profile_lengths[i][j]}"
+      end
+    end
+  end
+end
+
+def write_cluster_chromosome_data(cluster_data, cluster_chromosome_data_file, limit)
+  File.open(cluster_chromosome_data_file, 'w') do |f|
+    f.puts %w[cluster_id chr count].join("\t")
+    index = 0
+    last_id = cluster_data.first.first unless cluster_data.empty?
+    cluster_data.each do |cluster_id, patient_number, chr, count|
+      index += 1 if cluster_id != last_id 
+      break if index == limit
+      f.puts ["#{patient_number}_#{index}", chr, count].join("\t")
+      last_id = cluster_id
+    end
+  end
+end
+
+def write_coverage_data(coverage_to_plot, coverage_to_plot_file)
+  File.open(coverage_to_plot_file, 'w') do |f|
+    coverage_to_plot.each do |chr, position, freq|
+     f.puts "#{chr}\t#{position}\t#{freq}"
+   end
+  end
+end
+
+
+def write_detailed_hpo_profile_evaluation(suggested_childs, detailed_profile_evaluation_file, summary_stats)
+  CSV.open(detailed_profile_evaluation_file, "wb") do |csv|
+    suggested_childs.each do |pat_id, suggestions|
+      warning = nil
+      warning = 'WARNING: Very few phenotypes' if suggestions.length < 4
+      csv << ["PATIENT #{pat_id}", "#{warning}"]
+      csv << ["CURRENT PHENOTYPES", "PUTATIVE MORE SPECIFIC PHENOTYPES"]
+      suggestions.each do |parent, childs|
+        parent_code, parent_name = parent
+        if childs.empty?
+          csv << ["#{parent_name} (#{parent_code})", '-']
+        else
+          parent_writed = false
+          childs.each do |child_code, child_name|
+            if !parent_writed
+              parent_field = "#{parent_name} (#{parent_code})"
+              parent_writed = true
+            else
+              parent_field = ""
+            end
+            csv << [parent_field, "#{child_name} (#{child_code})"]
+          end
+        end
+      end
+      csv << ["", ""]
+    end
+  end
+end
+
+def write_arrays4scatterplot(x_axis_value, y_axis_value, filename, x_axis_name, y_axis_name)
+  File.open(filename, 'w') do |f|
+    f.puts "#{x_axis_name}\t#{y_axis_name}"
+    x_axis_value.each_with_index do |value,i|
+      y_value = y_axis_value[i]
+      raise("The #{i} position is not presented in y_axis_value") if y_value.nil?
+      f.puts [value, y_value].join("\t")
+    end
+  end
+end
+
+
+def write_similarity_matrix(similarity_matrix, similarity_matrix_file)  
+  File.open(similarity_matrix_file, 'w') do |f|
+    similarity_matrix.each do |row|
+      f.puts row.join("\t")
+    end
+  end
+end
+
+def write_profile_pairs(similarity_pairs, filename)
+  File.open(filename, 'w') do |f|
+    similarity_pairs.each do |pairsA, pairsB_and_values|
+      pairsB_and_values.each do |pairsB, values|
+        f.puts "#{pairsA}\t#{pairsB}\t#{values}"
+      end
+    end
+  end
+end
+
+def write_patient_hpo_stat(average_hp_per_pat_distribution, output_file)
+  File.open(output_file, 'w') do |f|
+    f.puts "#{'PatientsNumber'}\t#{'HPOAverage'}"
+    average_hp_per_pat_distribution.each do |patient_num, ave|
+      f.puts "#{patient_num}\t#{ave}"
+    end
+  end
+end
+
+def parse_clusters_file(clusters_file, patient_data)
+  clusters_info = {}
+  clusters_table = []
+  File.open(clusters_file).each do |line|
+    line.chomp!
+    patientID, clusterID = line.split("\t")
+    patientHPOProfile = patient_data.get_profile(patientID)
+    query = clusters_info[clusterID]
+    if query.nil? 
+      clusters_info[clusterID] = {patientID => patientHPOProfile}
+    else
+      query[patientID] = patientHPOProfile
+    end
+  end
+  clusters_info.each do |clusterID, patients_info|
+    patients_per_cluster = patients_info.keys.length
+    clusters_table << [clusterID, patients_per_cluster, patients_info.keys, patients_info.values]
+  end
+  return clusters_table, clusters_info
+end
+
+def load_profiles(file_path, hpo)
+  profiles = {}
+  #count = 0
+  File.open(file_path).each do |line|
+    id, profile = line.chomp.split("\t")
+    hpos = profile.split(',').map{|a| a.to_sym}
+    hpos, rejected_hpos = hpo.check_ids(hpos)
+    if !hpos.empty?
+      hpos = hpo.clean_profile(hpos)
+      profiles[id] = hpos if !hpos.empty?
+    end
+  end
+  return profiles
+end
+
+def load_variants(variant_folder)
+  variants = {}
+  Dir.glob(File.join(variant_folder, '*.tab')).each do |path|
+    profile_id = File.basename(path, '.tab')
+    vars = []
+    File.open(path).each do |line|
+      fields = line.chomp.split("\t")
+      chr = fields[0]
+      start = fields[1].to_i
+      vars << [chr, start, start]
+    end
+    variants[profile_id] = Genomic_Feature.new(vars)
+  end
+  return variants
+end
+
+def load_evidences(evidences_path, hpo)
+  genomic_coordinates = {}
+  coord_files = Dir.glob(File.join(evidences_path, '*.coords'))
+  coord_files.each do |cd_f|
+    entity = File.basename(cd_f, '.coords')
+    coordinates = load_coordinates(cd_f)
+    genomic_coordinates[entity] = coordinates
+  end
+  evidences = {}
+  evidence_files = Dir.glob(File.join(evidences_path, '*_HP.txt'))
+  evidence_files.each do |e_f|
+    pair = File.basename(e_f, '.txt')
+    profiles, id2label = load_evidence_profiles(e_f, hpo)
+    evidences[pair] = {prof: profiles, id2lab: id2label}
+  end
+  return evidences, genomic_coordinates
+end
+
+def load_coordinates(file_path)
+  coordinates = {}
+  header = true
+  File.open(file_path).each do |line|
+    fields = line.chomp.split("\t")
+    if header
+      header = false
+    else
+      entity, chr, strand, start, stop = fields
+      coordinates[entity] = [chr, start.to_i, stop.to_i, strand]
+    end
+  end
+  return coordinates
+end
+
+def load_evidence_profiles(file_path, hpo)
+  profiles = {}
+  id2label = {}
+  #count = 0
+  File.open(file_path).each do |line|
+    id, label, profile = line.chomp.split("\t")
+    hpos = profile.split(',').map{|a| a.to_sym}
+    hpos, rejected_hpos = hpo.check_ids(hpos)
+    if !hpos.empty?
+      hpos = hpo.clean_profile(hpos)
+      profiles[id] = hpos if !hpos.empty?
+      id2label[id] = label
+    end
+  end
+  return profiles, id2label
+end
+
+#Common methods for predictors
+#Training file example = 9  131371492   131375954   HP:0010974  2.41161970596 9.3.A.5
+#1. Indexing by chr (region)
+def coor_overlap?(ref_start, ref_stop, start, stop)
+  overlap = false
+  if (stop > ref_start && stop <= ref_stop) ||
+    (start >= ref_start && start < ref_stop) ||
+    (start <= ref_start && stop >= ref_stop) ||
+    (start > ref_start && stop < ref_stop)
+    overlap = true
+  end
+  return overlap
+end
+
+def load_training_file4regions(training_file)
+  training_set = {}
+  posInfo = loadFile(training_file)
+  posInfo.each do |info|
+    chr = info.shift
+    query = training_set[chr]
+    if query.nil?
+      training_set[chr] = [info]
+    else
+      query << info
+    end
+  end
+  return training_set
+end
+
+#2. Indexing by hpo (code)
+#prepare training file for analysis using phenotype2region prediction
+def load_training_file4HPO(training_file, thresold=0)
+  training_set = {}
+  information = loadFile(training_file, thresold)
+  information.each do |info|
+    hpoCode = info.delete_at(4)
+    query = training_set[hpoCode]
+    if query.nil?
+      training_set[hpoCode] = [info]  
+    else
+      query << info
+    end
+  end
+  # STDERR.puts training_set.keys.inspect
+  return training_set
+end
+
+
+#3. Load training info file:
+#Chr;Start;Stop;HPO;Association;node
+def loadFile(file, thresold=0)
+  information = []
+  File.open(file).each do |line|
+    line.chomp!
+    allInfo = line.split("\t")
+    associationValue = allInfo[4].to_f
+    if associationValue >= thresold
+      chr = allInfo[0]
+      startPos = allInfo[1].to_i
+      stopPos = allInfo[2].to_i
+      hpoCode = allInfo[3]
+      nodeID = allInfo[5]
+      information << [chr, startPos, stopPos, nodeID, hpoCode, associationValue]
+    end
+  end
+  return information
+end
+
+def load_hpo_ci_values(information_coefficient_file)
+  hpos_ci_values = {}
+  File.open(information_coefficient_file).each do |line|
+    line.chomp!
+    hpo_code, ci = line.split("\t")
+    hpos_ci_values[hpo_code.to_sym] = ci.to_f
+  end
+  return hpos_ci_values
+end
+
+def load_clustered_patients(file)
+  clusters = {}
+  File.open(file).each do |line|
+    line.chomp!
+    pat_id, cluster_id = line.split("\t")
+    query = clusters[cluster_id]
+    if query.nil?
+      clusters[cluster_id] = [pat_id]
+    else
+      query << pat_id
+    end
+  end
+  return clusters
+end
+
+def load_gene_data(gene_data_path)
+  gene_list = {} #geneID => attr
+  gene_location = {} # chr => gene
+  infile = open(gene_data_path)
+  gz = Zlib::GzipReader.new(infile)
+  current_chr = nil
+  genes = []
+  gz.each_line do |line|
+    line.chomp!
+    next if line =~ /^#/ 
+    fields = line.split("\t")
+    if fields[8].include?('genome=chromosome')
+      chr = fields[8].split(';')[1].split('=').last
+      gene_location[current_chr] = genes
+      genes = []
+      current_chr = chr
+    elsif fields[2] == 'gene'
+      attributes = {}
+      fields[8].split(';').each do |pair| 
+        key, value = pair.split('=')
+        attributes[key] = value
+      end
+      geneName = nil
+      geneName = attributes['gene'] if !attributes['gene'].nil?
+      geneSyns = []
+      geneSyns = attributes['gene_synonym'].split(',') if !attributes['gene_synonym'].nil?
+      description = attributes['description']
+      description = URI.unescape(description) if !description.nil?
+      attributes['Dbxref'] =~ /GeneID:(\d+)/
+      gene_list[$1] = [geneName, geneSyns, description]
+      genes << [$1, fields[3].to_i, fields[4].to_i]
+    end
+  end
+  gene_location[current_chr] = genes
+  return gene_list, gene_location
+end
+
+def parse_kegg_data(query_genes)
+  kegg_data = {} #gene => attb
+    while !query_genes.empty?
+      gene_set = query_genes.shift(10)
+      url = "http://rest.kegg.jp/get/#{gene_set.map{|qg| "hsa:#{qg}"}.join('+')}"
+      uri = URI(url)
+      response = Net::HTTP.get(uri) 
+    geneID = nil
+    gene_names = []
+    definition = nil
+    pathways = []
+    parsing_pathway_field = false
+    response.squeeze(' ').each_line do |line|
+      line.chomp!
+      if line =~ /^ENTRY/
+        geneID = line.split(' ')[1]
+      elsif line =~ /^NAME/
+        gene_names = line.split(' ', 2).last.split(', ')
+      elsif line =~ /^DEFINITION/
+        definition = line.split(' ', 2)[1]
+      elsif line =~ /^PATHWAY/
+        pathways << line.split(' ', 3)[1..2]
+        parsing_pathway_field = true
+      elsif line =~ /^BRITE/ || line =~ /^POSITION/ || line =~ /^DISEASE/ || line =~ /^MODULE/ || line =~ /^DRUG_TARGET/ || line =~ /^NETWORK/
+        parsing_pathway_field = false
+      elsif parsing_pathway_field
+        pathways << line.strip.split(' ', 2)
+      elsif line == '///'
+        parsing_pathway_field = false
+        kegg_data[geneID] = [gene_names, definition, pathways]
+        pathways = []
+        gene_names = []
+      end
+    end
+  end
+  return kegg_data
+end
+
+def write_compressed_plain_file(data, path)
+  File.open(path, 'w') do |f|
+    gz = Zlib::GzipWriter.new(f)
+    gz.write data.to_json
+    gz.close
+  end
+end
+
+def read_compressed_json(path)
+  infile = open(path)
+  gz = Zlib::GzipReader.new(infile)
+  object = JSON.parse(gz.read)
+  return object
+end
+
+def download(ftp_server, path, name)
+  ftp = Net::FTP.new()
+  ftp.connect(ftp_server)
+  ftp.login
+  ftp.getbinaryfile(path, name)
+  ftp.close
+end

data/lib/pets/parsers/cohort_parser.rb

@@ -0,0 +1,106 @@
+class Cohort_Parser
+	def self.load(options)
+		fields2extract = get_fields2extract(options)
+		field_numbers = fields2extract.values
+		records = read_records(options, fields2extract, field_numbers)
+		cohort, rejected_terms, rejected_recs = create_cohort(records, options)
+		return cohort, rejected_terms, rejected_recs
+	end 
+
+	def self.read_records(options, fields2extract, field_numbers)
+		records = {}
+		count = 0
+		File.open(options[:input_file]).each do |line|
+			line.chomp!
+			if options[:header] && count == 0
+				line.gsub!(/#\s*/,'') # correct comment like	headers
+				field_names = line.split("\t")
+				get_field_numbers2extract(field_names, fields2extract)
+				field_numbers = fields2extract.values
+			else
+				fields = line.split("\t")
+				record = field_numbers.map{|n| fields[n]}
+				if fields2extract[:id_col].nil?
+					id = "rec_#{count}" #generate ids
+				else
+					id = record.shift
+				end
+				if !record[0].nil?
+					record[0] = record[0].split(options[:separator])
+				else
+					record[0] = []
+				end
+				record[2] = record[2].to_i if !options[:start_col].nil?
+				record[3] = record[3].to_i if !options[:end_col].nil?
+				query = records[id]
+				if query.nil?
+					records[id] = [record]
+				else
+					query << record
+				end
+			end
+			count +=1
+		end
+		return records
+	end
+
+	def self.get_fields2extract(options)
+		fields2extract = {}
+		[:id_col, :ont_col, :chromosome_col, :start_col, :end_col].each do |field|
+			col = options[field]
+			if !col.nil?
+				col = col.to_i if !options[:header]
+				fields2extract[field] = col
+			end
+		end
+		return fields2extract
+	end
+
+	def self.get_field_numbers2extract(field_names, fields2extract)
+		fields2extract.each do |field, name|
+			fields2extract[field] = field_names.index(name)
+		end
+	end
+
+	def self.create_cohort(records, options)
+		ont = Cohort.get_ontology(Cohort.act_ont)
+		rejected_terms = []
+		rejected_recs = []
+		cohort = Cohort.new()
+		records.each do |id, record|
+			rec = record.first
+			terms = rec.first
+			if options[:names]
+				init_term_number = terms.length
+				terms, rec_rejected_terms = ont.translate_names(terms)
+				if !rec_rejected_terms.empty?
+					STDERR.puts "WARNING: record #{id} has the unknown term NAMES '#{rec_rejected_terms.join(',')}'. Terms removed."
+					rejected_terms.concat(rec_rejected_terms)
+				end
+				if terms.empty? && init_term_number > 0
+					rejected_recs << id
+					next
+				end
+			end
+			if rec.length > 1 # there is genomic region attributes
+				variants = record.map{|v| v[1..3] }
+			else
+				variants = [] # Not exists genomic region attributes so we create a empty array
+			end
+			cohort.add_record([id, terms, check_variants(variants)])
+		end
+		return cohort, rejected_terms.uniq, rejected_recs
+	end
+
+	def self.check_variants(vars)
+		checked_vars = []
+		vars.each do |var| #[chr, start, stop]
+			if var.first == '-' # the chr must be defined
+				STDERR.puts "WARNING: variant #{var.join(',')} has been removed"
+			else
+				checked_vars << var
+			end
+		end
+		return vars
+	end
+end

data/lib/pets/version.rb

@@ -1,3 +1,3 @@
 module Pets
-  VERSION = "0.2.3"
+  VERSION = "0.2.4"
 end