anncrsnp 0.1.0

data/bin/grdbmanager.rb

@@ -0,0 +1,226 @@
+#! /usr/bin/env ruby
+
+ROOT_PATH = File.dirname(__FILE__)
+$: << File.expand_path(File.join(ROOT_PATH, '..', 'lib', 'anncrsnp'))
+$: << File.expand_path(File.join(ROOT_PATH, '..', 'lib', 'anncrsnp', 'parsers'))
+
+require 'optparse'
+require 'ucscparser'
+require 'dataset'
+require 'sqlite3'
+
+options = {}
+OptionParser.new do |opts|
+  opts.banner = "Usage: #{__FILE__} [options]"
+
+  options[:data] = nil
+  opts.on("-d", "--data_directory PATH", "Directory used to extract data") do |data|
+    options[:data] = data
+  end
+
+  options[:create_sql] = FALSE
+  opts.on("-s", "--create_sql", "Create SQL DB") do
+    options[:create_sql] = TRUE
+  end
+
+  options[:output_path] = "genomic_data.sqlite"
+  opts.on("-o", '--output_path PATH', 'Output path for DB') do |output_path|
+  	options[:output_path] = output_path
+  end
+
+  options[:verbose] = nil
+  opts.on("-v", "--[no-]verbose", "Run verbosely") do |v|
+    options[:verbose] = v
+  end
+
+end.parse!
+
+
+
+all_data = {}
+if File.exist?(options[:data])
+	Dir.glob(File.join(options[:data],'*.{txt,bed,csv}')).each do |file| # we get the path to each file on directory
+		current_file = File.basename(file)
+### Definitive sources
+#If bin field from UCSC doesn't exist, put FALSE as input data to parseUCSCformat method
+		if current_file == "wgEncodeAwgDnaseMasterSites.bed"
+			header = [:score, :floatScore, :sourceCount, :sourceIds]
+			current_dataset = parseUCSCformat(file, header, FALSE)
+			current_dataset.numeric_filter(:sourceCount, 2)
+			current_dataset.drop_columns(header)
+			current_dataset.add_metadata(:classification, 'DNAseHS')
+			all_data['dnaseData'] = current_dataset			
+		elsif current_file == "wgEncodeHaibMethyl450Ag04449SitesRep1.bed"
+			header = [:score, :strand, :thickStart, :thickEnd, :itemRgb]
+			current_dataset = parseUCSCformat(file, header, FALSE)
+			current_dataset.drop_columns(header)
+			current_dataset.add_metadata(:classification, 'Metilation_sites')
+			all_data['metilationData'] = current_dataset
+		elsif current_file == "snp144Common.txt" # current_file == "test.txt"
+			header = [:score, :strand, :refNCBI, :refUCSC, :observed, :molType, :class, :valid, :avHet, :avHetSE, :func, :locType, :weight, :exceptions, :submitterCount, :submitters, :alleleFreqCount, :alleles, :alleleNs, :alleleFreqs, :bitfields]
+			current_dataset = parseUCSCformat(file, header)
+			current_dataset.drop_columns([:score, :strand, :refNCBI, :refUCSC, :observed, :molType, :valid, :avHet, :avHetSE, :locType, :weight, :exceptions, :submitterCount, :submitters, :alleleFreqCount, :alleles, :alleleNs, :alleleFreqs, :bitfields])
+			current_dataset.add_metadata(:classification, 'SNP')
+			all_data['snpDbSnp'] = current_dataset
+		elsif current_file == "refGene.txt" 
+			header = [:name, :strand, :cdsStart, :cdsEnd, :exonCount, :exonStarts, :exonEnds, :score, :cdsStartStat, :cdsEndStat, :exonFrames]
+			current_dataset = parseUCSCrefseqformat(file, header)
+			current_dataset.drop_columns(header)
+			current_dataset.add_metadata(:classification, 'gene')
+			all_data['gene'] = current_dataset
+		elsif current_file == "TFBSMasterSites.txt" #Must be generated with "masterfeatures.rb tfbs/files.txt antibody import_data/TFBSMasterSites.txt tfbs/"
+			header = []
+			current_dataset = parseUCSCformat(file, header, FALSE)
+			current_dataset.add_metadata(:classification, 'TFBS')
+			all_data['tfbs'] = current_dataset
+		elsif current_file == "HistoneModMasterSites.txt" #Must be generated with "masterfeatures.rb tfbs/files.txt antibody import_data/TFBSMasterSites.txt tfbs/"
+			header = []
+			current_dataset = parseUCSCformat(file, header, FALSE)
+			current_dataset.add_metadata(:classification, 'HistoneModification')
+			all_data['HistoneModification'] = current_dataset				
+		elsif current_file == "46waycons.txt" 
+			header = [:span, :count, :offset, :file, :lowerLimit, :dataRange, :validCount, :sumData, :sumSquares]
+			current_dataset = parseUCSCformat(file, header)
+			current_dataset.drop_columns(header)
+			current_dataset.add_metadata(:classification, 'ConservedRegions')
+			all_data['ConservedRegions'] = current_dataset
+		elsif current_file == "enhancer_tss_associations.bed" 
+			header = [:score, :strand, :enh_start, :enh_stop, :array, :index, :val1, :val2]
+			current_dataset = parseUCSCformat(file, header, FALSE)
+			current_dataset.drop_columns(header)
+			current_dataset.add_metadata(:classification, 'Enhancers')
+			all_data['Enhancers'] = current_dataset
+		elsif current_file == "enhancers.csv" 
+			header = [:cell_line, :index1, :index2, :index3, :index4, :index5, :index6, :index7]
+			current_dataset = parseDENdbCSVformat(file, header)
+			current_dataset.drop_columns(header)
+			current_dataset.add_metadata(:classification, 'DENdbEnhancers')
+			all_data['DENdbEnhancers'] = current_dataset
+		elsif current_file == "all_hg19_bed.bed" 
+			header = [:counter]
+			current_dataset = parseUCSCformat(file, header, FALSE)
+			current_dataset.drop_columns(header)
+			current_dataset.add_metadata(:classification, 'SuperEnhancers')
+			all_data['SuperEnhancers'] = current_dataset
+		end
+	end
+end
+
+if options[:create_sql]
+	commands = []
+	if !File.exists?(options[:output_path])
+		commands << "CREATE TABLE GenomicRange(
+			bin,
+			chr,
+			start,
+			end,
+			type,
+			name,
+			AnnotationId
+		)"
+	 	commands << "CREATE TABLE Annotation(
+			value,
+			AnnotationTypeId
+		)"
+		commands << "CREATE TABLE AnnotationType(
+			type
+		)"
+		#File.delete(options[:output_path]) 
+	end
+
+	DB = SQLite3::Database.new( options[:output_path] )
+	commands.each do |cmd|
+		DB.execute(cmd)
+	end
+	# Import data process speed up configuration
+	DB.execute("PRAGMA synchronous = OFF;")
+	DB.execute("PRAGMA journal_mode = MEMORY;")
+
+	# Creating memory indexes for incremental updates
+	annotation_type_index = DB.execute("SELECT rowid, * FROM AnnotationType").group_by {|r| r[1]}
+	annotation_index = DB.execute("SELECT rowid, * FROM Annotation").group_by {|r| r[1]}
+
+	all_data.each do |class_data, dataset|
+		puts "#{class_data} import started"
+		# Save and create AnnotationType data
+		#------------------------------------------------------------------
+		header = dataset.get_metadata(:header)
+		annotation_type = header.map{|h| h.to_s}
+		annotation_type.shift(4)
+		if !commands.empty?
+			records = annotation_type
+		else
+			records = annotation_type.select{|at| annotation_type_index[at].first.nil?} 
+		end
+
+		DB.transaction do |db|
+			db.prepare("INSERT INTO AnnotationType(type) VALUES(?)") do |smnt| # Precompile query for speed up process
+				records.each do |rec|
+					smnt.execute(rec)
+				end
+			end
+		end		
+		annotation_type_index = DB.execute("SELECT rowid, * FROM AnnotationType").group_by {|r| r[1]} if !records.empty?
+
+		# Save and create AnnotationType data
+		#------------------------------------------------------------------
+		annotations = {}
+		annotation_type.each do |at|
+			annotations[at] = {}
+		end
+		if dataset.first.length > 4
+			dataset.each_record do |record|
+				record[5..record.length - 1].each_with_index do |annotation, i|
+						annotations[annotation_type[i]][annotation] = nil
+				end
+			end
+		end
+
+		records = []
+		annotations.each do |annotation_type, values|
+			annotation_type_id = annotation_type_index[annotation_type].first.first
+			if !commands.empty?
+				records = records.concat(values.keys.map{|v| [v, annotation_type_id]})
+			else
+				records = records.concat(values.keys.select{|v| annotation_index[v].nil? }.map{|v| [v, annotation_type_id]}) 
+			end
+		end
+
+		DB.transaction do |db|
+			db.prepare("INSERT INTO Annotation(value, AnnotationTypeId) VALUES(?, ?)") do |smnt|
+				records.each do |rec|
+					smnt.execute(rec[0], rec[1])
+				end
+			end
+		end		
+		annotation_index = DB.execute("SELECT rowid, * FROM Annotation").group_by {|r| r[1]} if !records.empty?
+
+		# Save and create GenomicRange data
+		#------------------------------------------------------------------
+		DB.transaction do |db|
+			db.prepare("INSERT INTO GenomicRange(bin, chr, start, end, type, name, AnnotationId) VALUES(?, ?, ?, ?, ?, ?, ?)") do |smnt|
+				dataset.each_record do |record|
+					region_data = record.shift(4)
+					annotation_ids = []
+					record.each do |annotation|
+						id = annotation_index[annotation]
+						annotation_ids << id.first.first if !id.nil?
+					end
+					smnt.execute(
+						region_data[1]/10000,
+						region_data[0],
+						region_data[1],
+						region_data[2],
+						class_data,
+						region_data[3],
+						annotation_ids.join(',')
+						)
+				end
+			end
+		end
+		puts "#{class_data} import finished"
+	end
+end
+DB.execute("CREATE INDEX name_index ON GenomicRange (name)")
+DB.execute("CREATE INDEX bin_index ON GenomicRange (bin)")
+DB.close

data/bin/masterfeatures.rb

@@ -0,0 +1,188 @@
+#! /usr/bin/env ruby
+
+require 'benchmark'
+
+#Description
+#--------------
+#Tool to unify data from ENCODE
+
+#Methods
+#--------------
+
+def load_metadata_file(file)
+	name_storage = {}
+	file_text = File.open(file)
+	file_text.each do |line|
+		line.chomp!
+		fields = line.split("\t")
+		features_storage = {} #metadata hash
+		features = fields[1].split("; ")
+		features.each do |feature|
+			met_name, metadata = feature.split("=")
+			features_storage[met_name] = metadata
+		end
+		name_storage[fields[0].gsub('.gz', '')] = features_storage
+	end
+	file_text.close
+	return name_storage
+end
+
+def element_grouper(grouping_element, name_storage) #erase redundance by antibody by default
+	package_grouping = {}
+	name_storage.each do |file_name, metadata|
+		selected_element = metadata[grouping_element]
+		if !selected_element.nil?	 #verify is there is an element in this field (avoid mistakes).
+			if !package_grouping[selected_element].nil?
+				package_grouping[selected_element] << file_name
+			else
+				package_grouping[selected_element] = [file_name]
+			end
+		end
+	end
+	return package_grouping
+end
+
+def load_files_to_compare(file_name)
+	genomic_regions = {}
+	File.open(file_name).each do |line|
+		line.chomp!
+		genomic_info = line.split("\t")
+		chr = genomic_info.shift
+		genomic_info = genomic_info[0..1].map{|c| c.to_i}
+		bin = genomic_info.first/10000
+		query = genomic_regions[chr]
+		if query.nil?
+			genomic_regions[chr] = {bin => [genomic_info]}
+		else
+			query_bin = query[bin]
+			if query_bin.nil?
+				query[bin] = [genomic_info]
+			else
+				query_bin << genomic_info
+			end
+		end
+	end
+	return genomic_regions
+end
+
+def compare_genomics_regions(main_genomic_regions, genomic_regions_to_compare, thresold_overlap)
+	selected_genomic_regions = {}
+	genomic_regions_to_compare.each do |chr_reg, genomic_region_to_compare|
+		genomic_region_to_compare.each do |bin, regs|
+			batch_match = false
+			query_main = main_genomic_regions[chr_reg] # main_genomic_regions has chr_reg?
+			if !query_main.nil? # main_genomic_regions has chr_reg!
+				query_main_bin = query_main[bin]
+				if !query_main_bin.nil?
+					batch_match = true
+					regs.each do |reg|
+						match = false
+						query_main_bin.each do |main|
+							match = compare_genomics_regions_coords(main, reg, thresold_overlap)
+							break if match
+						end
+						save_reg(selected_genomic_regions, chr_reg, bin, reg) if !match
+					end
+				end
+			end
+			if !batch_match
+				regs.each do |reg|
+					save_reg(selected_genomic_regions, chr_reg, bin, reg)
+				end
+			end
+		end
+	end
+	return selected_genomic_regions
+end
+
+def save_reg(selected_genomic_regions, chr_reg, bin, reg)
+	query_chr = selected_genomic_regions[chr_reg]
+	if !query_chr.nil?
+		query_bin = query_chr[bin]
+		if  query_bin.nil?
+			query_chr[bin] = [reg]
+		else
+			query_bin << reg
+		end
+	else
+		selected_genomic_regions[chr_reg] = {bin => [reg]}
+	end
+end
+
+def save_reg_concat(selected_genomic_regions, chr_reg, bin, reg)
+	query_chr = selected_genomic_regions[chr_reg]
+	if !query_chr.nil?
+		query_bin = query_chr[bin]
+		if  query_bin.nil?
+			query_chr[bin] = reg
+		else
+			query_bin.concat(reg)
+		end
+	else
+		selected_genomic_regions[chr_reg] = {bin => reg}
+	end
+end
+
+def compare_genomics_regions_coords(main_genomic_region, genomic_region_to_compare, thresold_overlap)
+	match = false
+	
+	main_beg, main_end = main_genomic_region
+	reg_beg, reg_end = genomic_region_to_compare
+	size_main_genomic_region = main_end - main_beg
+	size_genomic_region_to_compare = reg_end - reg_beg
+
+	absolute_overlap = 0
+	if reg_beg >= main_beg && reg_beg <= main_end
+		absolute_overlap = main_end - reg_beg
+	elsif reg_end >= main_beg && reg_end <= main_end
+		absolute_overlap = reg_end - main_beg
+	elsif reg_beg <= main_beg && reg_end >= main_end 
+		absolute_overlap = size_main_genomic_region
+	elsif reg_beg >= main_beg && reg_end <= main_end 
+		absolute_overlap = size_genomic_region_to_compare
+	end		
+	main_relative_overlap = absolute_overlap / size_main_genomic_region * 1.0
+	compare_relative_overlap = absolute_overlap / size_genomic_region_to_compare * 1.0 
+	if main_relative_overlap >= thresold_overlap || compare_relative_overlap >= thresold_overlap
+		match = true
+	end
+
+	return match
+end
+
+#Main
+#--------------
+file_input_folder = ARGV[3]
+name_storage = load_metadata_file(ARGV[0])
+package_grouping = element_grouper(ARGV[1], name_storage)
+file_writer = File.open(ARGV[2],'w')
+
+package_grouping.each do |grouping_element, file_names|
+	#abrir el primer archivo del paquete
+	genomic_regions_references = load_files_to_compare(File.join(file_input_folder, file_names.shift))
+	file_names.each do |f_name|
+		file2compare = load_files_to_compare(File.join(file_input_folder, f_name))
+		selected_genomic_regions = selected_genomic_regions = compare_genomics_regions(genomic_regions_references, file2compare, 0.8)
+		#puts Benchmark.measure{selected_genomic_regions = compare_genomics_regions(genomic_regions_references, file2compare, 0.8)}
+		selected_genomic_regions.each do |chr, ge_regs|
+			ge_regs.each do |bin, reg|
+				save_reg_concat(genomic_regions_references, chr, bin, reg)
+			end
+		end		
+	end
+	genomic_regions_references.each do |chr, ge_regs|
+		ge_regs.each do |bin, reg|
+			reg.each do |gr|
+				file_writer.puts "#{chr}\t#{gr.join("\t")}\t#{grouping_element}"
+			end
+		end
+	end
+	puts "Wrote #{grouping_element}"
+end
+
+file_writer.close
+
+
+
+
+

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/bin/statistics.rb

@@ -0,0 +1,193 @@
+#! /usr/bin/env ruby
+require 'scbi_plot'
+#METHODS
+#----------
+def load_snp_data(input_file, fields_length)
+	snp_storage = {}
+	index = {}
+	counter = 0
+	File.open(input_file).each do |line|
+		line.chomp!
+		fields = line.split("\t")
+		snp_fields = fields.shift(fields_length) #in fields you store the genomic factors (histone modif, tfbs...)
+		if counter == 0
+			fields.each_with_index do |category, position|
+				index[category] = position
+			end
+		else
+			categories = {
+							"HistoneModification" => [], 
+							"tfbs" => [], 
+							"dnaseData" => [],
+							"metilationData" => [],
+							"ConservedRegions" => [],
+							"Enhancers" => [],
+							"DENdbEnhancers" => [], 
+							"SuperEnhancers" => []
+						}
+			categories.each do |category_name, category_value|
+				column_position = index[category_name]
+				if !column_position.nil?
+					snp_category_values = fields[column_position]
+					category_value.concat(snp_category_values.split(',')) if snp_category_values != '-' 
+				end	
+			end
+			snp_storage[snp_fields[0]] = categories
+					
+			#bloque de código para tratar la información de los snps
+		end
+		counter += 1
+	end
+	return snp_storage
+end
+
+def snp_calculate_stats(snp_storage)
+	snp_percentage = {
+							"HistoneModification" => 0, 
+							"tfbs" => 0, 
+							"dnaseData" => 0,
+							"metilationData" => 0,
+							"ConservedRegions" => 0,
+							"Enhancers" => 0,
+							"DENdbEnhancers" => 0,
+							"SuperEnhancers" => 0
+						}
+	snp_storage.each do |snp_name, annotations|
+		annotations.each do |annotation_category, annotation_value|
+			if !annotation_value.empty?
+				snp_percentage[annotation_category] += 1 #possible error point! 	
+			end
+		end
+	end
+	total_snps = snp_storage.length.to_f
+	snp_percentage.each do |annotation_category, true_positive_number|
+		percentage = true_positive_number / total_snps * 100
+		snp_percentage[annotation_category] = percentage
+	end
+	return snp_percentage
+end
+
+def create_histogram(snp_percentage, name)
+	# create Histogram
+	p=ScbiPlot::Histogram.new(name,'SNPs genomic region annotations')
+
+	# add x axis data
+
+	p.add_x(snp_percentage.keys)
+	puts snp_percentage.keys.inspect
+	# add y axis data
+	p.add_y(snp_percentage.values)
+	puts snp_percentage.values.inspect
+	# generate graph
+	p.do_graph
+end
+
+
+def snp_calculate_stats_with_reference(snp_storage, snp_storage_reference)
+	snp_percentage = {
+							"HistoneModification" => 0, 
+							"tfbs" => 0, 
+							"dnaseData" => 0,
+							"metilationData" => 0,
+							"ConservedRegions" => 0,
+							"Enhancers" => 0,
+							"DENdbEnhancers" => 0,
+							"SuperEnhancers" => 0
+						}
+
+	snp_storage_reference.each do |snp_name_ref, annotations_ref|
+		query = snp_storage[snp_name_ref]
+		if !query.nil?
+			annotations_ref.each do |annotation_category_ref, annotation_value_ref|
+				annotation_value = query[annotation_category_ref]
+				if annotation_comparison(annotation_value_ref, annotation_value, annotation_category_ref)
+					snp_percentage[annotation_category_ref] += 1 	
+				end
+			end
+		end
+	end
+
+	total_snps = snp_storage_reference.length.to_f
+	snp_percentage.each do |annotation_category, true_positive_number|
+		percentage = true_positive_number / total_snps * 100
+		snp_percentage[annotation_category] = percentage
+	end
+	return snp_percentage
+end
+
+def annotation_comparison(annotation_value_ref, annotation_value, annotation_category_ref)
+	result = false
+	annotation_value_ref.uniq!
+	annotation_value.uniq!
+	#puts "#{annotation_value_ref.inspect} => #{annotation_value}" if annotation_category_ref == 'dnaseData'
+	if annotation_value_ref.sort == annotation_value.sort 
+		result = true
+	elsif annotation_category_ref == 'dnaseData' && 
+		!annotation_value.empty? 
+		result = true
+	elsif annotation_category_ref == 'tfbs'	
+		if !(annotation_value_ref & annotation_value).empty? || annotation_value.length >= 5 			
+			result= true
+		end
+	elsif annotation_category_ref == 'metilationData' &&
+		!annotation_value.empty?
+		result = true	
+	elsif annotation_category_ref == 'HistoneModification'
+		annotation_value_ref = annotation_value_ref.map{|an| 
+			if /(H\d+K\d+)\w*/ =~ an
+				$1
+			else
+				an
+			end
+		}.uniq
+		annotation_value = annotation_value.map{|an| 
+			if /(H\d+K\d+)\w*/ =~ an
+				$1
+			else
+				an
+			end
+		}.uniq
+		if !(annotation_value_ref & annotation_value).empty? || annotation_value.length >= 5 			
+			result= true
+		end	
+	elsif annotation_category_ref == 'ConservedRegions' && 
+		!annotation_value.empty? 
+		result = true
+	elsif annotation_category_ref == 'Enhancers' &&
+		!annotation_value.empty?
+		result = true
+	elsif annotation_category_ref == 'DENdbEnhancers' &&
+		!annotation_value.empty?
+		result = true
+	elsif annotation_category_ref == 'SuperEnhancers' &&
+		!annotation_value.empty?
+		result = true
+	end
+	return result 
+end
+
+#MAIN
+#----------
+
+#RECUERDA: este programa hace analisis estadisticos y compara resultados para dos archivos dados.
+#En nuestro caso, comparamos los datos dados por nuestro programa con los datos obtenidos experimentalmente.
+#nuestros datos = ARGV[0], datos del experimento = ARGV[1]
+#si no se especifica segundo argumento de entrada = se hace el análisis sobre el propio resultado del programa
+fields_length = 5
+fields_length = ARGV[2].to_i if !ARGV[2].nil?
+
+snp_storage = load_snp_data(ARGV[0], fields_length)
+if !ARGV[1].nil? && ARGV[1].downcase != 'false'
+	snp_storage_reference = load_snp_data(ARGV[1])
+	snp_percentage = snp_calculate_stats_with_reference(snp_storage, snp_storage_reference) 
+else
+	snp_percentage = snp_calculate_stats(snp_storage)
+end
+snp_percentage.each do |category_name, percentage|
+	puts "#{category_name}\t#{percentage}\t#{ARGV[3]}"
+end
+
+#El archivo de graficado aparecera donde se ejecute el script
+# file_name = File.basename(ARGV[0], ".txt")
+# graph_name = file_name + ".png"
+# create_histogram(snp_percentage, graph_name)