bio-publisci 0.0.1

data/lib/bio-publisci/loader.rb

@@ -0,0 +1,36 @@
+# This is temporary, just to help w/ development so I don't have to rewrite r2rdf.rb to be
+# a standard gem base yet. Also load s the files instead of require for easy reloading
+require 'tempfile'
+require 'rdf'
+require 'csv'
+require 'rserve'
+require 'sparql'
+require 'sparql/client'
+require 'rdf/turtle'
+
+def load_folder(folder)
+	Dir.foreach(File.dirname(__FILE__) + "/#{folder}") do |file|
+		unless file == "." or file == ".."
+			load File.dirname(__FILE__) + "/#{folder}/" + file
+		end
+	end
+end
+
+load File.dirname(__FILE__) + '/dataset/interactive.rb'
+load File.dirname(__FILE__) + '/query/query_helper.rb'
+load File.dirname(__FILE__) + '/parser.rb'
+load File.dirname(__FILE__) + '/r_client.rb'
+load File.dirname(__FILE__) + '/analyzer.rb'
+load File.dirname(__FILE__) + '/store.rb'
+load File.dirname(__FILE__) + '/dataset/data_cube.rb'
+
+
+load_folder('metadata')
+load_folder('readers')
+load_folder('writers')
+load_folder('dataset/ORM')
+# Dir.foreach(File.dirname(__FILE__) + '/generators') do |file|
+# 	unless file == "." or file == ".."
+# 		load File.dirname(__FILE__) + '/generators/' + file
+# 	end
+# end

data/lib/bio-publisci/metadata/metadata.rb

@@ -0,0 +1,105 @@
+class String
+  def unindent
+    gsub /^#{self[/\A\s*/]}/, ''
+  end
+end
+
+module R2RDF
+  module Metadata
+    def defaults
+    {
+      encode_nulls: false,
+      base_url: "http://www.rqtl.org",
+    }
+    end
+
+    def basic(fields, options={} )
+      #TODO don't assume base dataset is "ns:dataset-var", 
+      #make it just "var", and try to make that clear to calling classes
+
+      fields[:var] = sanitize([fields[:var]]).first
+      options = defaults().merge(options)
+      str = <<-EOF.unindent
+      ns:dataset-#{fields[:var]} rdfs:label "#{fields[:title]}";
+        dct:title "#{fields[:title]}";
+        dct:creator "#{fields[:creator]}";
+        rdfs:comment "#{fields[:description]}";
+        dct:description "#{fields[:description]}";
+        dct:issued "#{fields[:date]}"^^xsd:date;
+      EOF
+
+      end_str = ""
+
+      if fields[:subject] && fields[:subject].size > 0
+        str << "\tdct:subject \n"
+        fields[:subject].each{|subject| str << "\t\t" + subject + ",\n" }
+        str[-2] = ";"
+      end
+
+      if fields[:publishers]
+        fields[:publishers].map{|publisher|
+          raise "No URI for publisher #{publisher}" unless publisher[:uri]
+          raise "No label for publisher #{publisher}" unless publisher[:label]
+          str << "\tdct:publisher <#{publisher[:uri]}> ;\n"
+          end_str << "<#{publisher[:uri]}> a org:Organization, foaf:Agent;\n\trdfs:label \"#{publisher[:label]}\" .\n\n"
+        }
+        str[-2] = '.'
+      end
+
+      str + "\n" + end_str
+    end
+
+    def provenance(fields, options={})
+      #TODO: should either add a prefixes method or replace some with full URIs
+      var = sanitize([fields[:var]]).first
+      source_software = fields[:software] # software name, object type, optionally steps list for, eg, R
+
+      str = "qb:dataset-#{var} a prov:Entity.\n"
+      endstr = "qb:dataset-#{var} prov:wasGeneratredBy <#{options[:base_url]}/ns/R2RDF>\n" #replace once gem has an actual name
+      if source_software
+        source_software = [source_software] unless source_software.respond_to? :map
+        source_software.map{|soft|
+          str << "<#{options[:base_url]}/ns/prov/software/#{soft}> a prov:Entity .\n"
+
+          #Note: probably should say derived from the software object, then software object from software.
+          endstr << "qb:dataset-#{var} prov:wasDerivedFrom <#{options[:base_url]}/ns/prov/#{soft}> .\n"
+        }
+      end
+    end
+
+    def r2rdf_metadata
+      str <<-EOF.unindent
+      <#{options[:base_url]}/ns/R2RDF> a foaf:Agent;
+        foaf:name "R2RDF Semantic Web Toolkit";
+        org:memberOf <http://sciruby.com/>
+      EOF
+    end
+
+    def org_metadata
+      str <<-EOF.unindent
+        <http://sciruby.com/> a org:Organization;
+          skos:prefLabel "SciRuby";
+          rdfs:description "A Project to Build and Improve Tools for Scientific Computing in Ruby".
+      EOF
+    end
+
+    def metadata_help(topic=nil)
+      if topic
+        puts "This should display help information for #{topic}, but there's none here yet :("
+      else
+        puts <<-EOF.unindent
+        Available metadata fields:
+        (Field)         (Ontology)                              (Description)
+
+        publishers      dct/foaf/org        The Organization/s responsible for publishing the dataset
+        subject         dct                 The subject of this dataset. Use resources when possible
+        var             dct                 The name of the datset resource (used internally)
+        creator         dct                 The person or process responsible for creating the dataset
+        description     dct/rdfs            A descriptions of the dataset
+        issued          dct                 The date of issuance for the dataset
+
+        EOF
+      end
+    end
+  end
+end

data/lib/bio-publisci/parser.rb

@@ -0,0 +1,64 @@
+module R2RDF
+	module Parser
+		def create_graph(string)
+			f = Tempfile.new('graph')
+			f.write(string)
+			f.close
+			graph = RDF::Graph.load(f.path, :format => :ttl)
+			f.unlink
+			graph
+		end
+
+		def get_ary(query_results,method='to_s')
+      query_results.map{|solution|
+        solution.to_a.map{|entry|
+          if entry.last.respond_to? method
+	          entry.last.send(method)
+	        else
+	        	entry.last.to_s
+	        end
+        }
+      }
+    end
+
+    def get_hashes(query_results,method=nil)
+    	arr=[]
+    	query_results.map{|solution|
+    		h={}
+    		solution.map{|element|
+					if method && element[1].respond_to?(method)
+					 	h[element[0]] = element[1].send(method)
+					else
+					 	h[element[0]] = element[1]
+					end
+    		}
+    		arr << h
+    	}
+    	arr
+    end
+
+    def observation_hash(query_results,shorten_uris=false,method='to_s')
+    	h={}
+    	query_results.map{|sol|
+    		(h[sol[:observation].to_s] ||= {})[sol[:property].to_s] = sol[:value].to_s
+    	}
+
+    	if shorten_uris
+	    	newh= {}
+	    	h.map{|k,v| 
+	    		newh[strip_uri(k)] ||= {}
+	    		v.map{|kk,vv| 
+	    			newh[strip_uri(k)][strip_uri(kk)] = strip_uri(vv)
+	    		}
+	    	}
+	    	newh
+	    else
+	    	h
+	    end
+    end
+
+    def strip_uri(uri)
+    	uri.to_s.split('/').last.split('#').last
+    end
+	end
+end

data/lib/bio-publisci/query/query_helper.rb

@@ -0,0 +1,114 @@
+module RDF
+	class Query
+		class Solutions
+			def to_h
+				arr=[]
+	    	self.map{|solution|
+	    		h={}
+	    		solution.map{|element|
+						 	h[element[0]] = element[1]
+	    		}
+	    		arr << h
+	    	}
+	    	arr
+			end
+		end
+	end
+end
+
+module R2RDF
+  #.gsub(/^\s+/,'')
+  module Query
+    def vocabulary
+      {
+        base: RDF::Vocabulary.new('<http://www.rqtl.org/ns/#>'),
+        qb:   RDF::Vocabulary.new("http://purl.org/linked-data/cube#"),
+        rdf:  RDF::Vocabulary.new('http://www.w3.org/1999/02/22-rdf-syntax-ns#'),
+        rdfs: RDF::Vocabulary.new('http://www.w3.org/2000/01/rdf-schema#'),
+        prop: RDF::Vocabulary.new('http://www.rqtl.org/dc/properties/'),
+        cs:   RDF::Vocabulary.new('http://www.rqtl.org/dc/cs')
+      }
+    end
+
+    
+
+    # def execute_internal(query,repo)
+    #   SPARQL.execute(query,repo)
+    # end
+
+    def execute(string,store,type=:fourstore)
+    	if type == :graph || store.is_a?(RDF::Graph) || store.is_a?(RDF::Repository)    		
+				sparql = SPARQL::Client.new(store)
+			elsif store.is_a? R2RDF::Store
+				sparql = SPARQL::Client.new(store.url+"/sparql/")				
+			elsif type == :fourstore
+				sparql = SPARQL::Client.new(store+"/sparql/")				
+      end
+      sparql.query(string)
+    end
+
+    def execute_from_file(file,store,type=:fourstore)
+      if File.exist?(file)
+        string = IO.read(file)
+      elsif File.exist?(File.dirname(__FILE__) + '/../../../resources/queries/' + file)
+        string = IO.read(File.dirname(__FILE__) + '/../../../resources/queries/' + file)
+      elsif File.exist?(File.dirname(__FILE__) + '/../../../resources/queries/' + file + '.rq')
+        string = IO.read(File.dirname(__FILE__) + '/../../../resources/queries/' + file + '.rq')
+      else
+        raise "couldn't find query for #{file}"
+      end
+    	execute(string, store, type)
+    end
+
+#     def prefixes
+#       <<-EOF
+# PREFIX ns:     <http://www.rqtl.org/ns/#> 
+# PREFIX qb:   <http://purl.org/linked-data/cube#> 
+# PREFIX rdf:  <http://www.w3.org/1999/02/22-rdf-syntax-ns#> 
+# PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#> 
+# PREFIX prop: <http://www.rqtl.org/dc/properties/> 
+# PREFIX cs:   <http://www.rqtl.org/dc/cs/>
+# PREFIX skos: <http://www.w3.org/2004/02/skos/core#> 
+
+#       EOF
+#     end
+
+    def property_values(var, property)
+      str = prefixes
+      str << <<-EOS
+SELECT ?val WHERE {
+  ?obs qb:dataSet ns:dataset-#{var} ;
+      prop:#{property} ?val ;
+}
+      EOS
+      str
+    end
+
+    def row_names(var)
+      str = prefixes
+      str << <<-EOS
+SELECT ?label WHERE {
+  ?obs qb:dataSet ns:dataset-#{var} ;
+       prop:refRow ?row .
+  ?row skos:prefLabel ?label .
+}
+      EOS
+    end
+
+    # Currently will say "___ Component", needs further parsing
+    def property_names(var)
+      str = prefixes
+      str << <<-EOS
+SELECT DISTINCT ?label WHERE {
+  ns:dsd-#{var} qb:component ?c .
+  ?c rdfs:label ?label
+}
+      EOS
+    end
+
+  end
+
+  class QueryHelper
+    include R2RDF::Query
+  end  
+end

data/lib/bio-publisci/r_client.rb

@@ -0,0 +1,54 @@
+module R2RDF
+	module Rconnect
+		
+		def connect(address=nil)
+			if address
+				Rserve::Connection.new(address)
+			else
+				Rserve::Connection.new
+			end
+		end
+
+		def load_workspace(connection,loc=Dir.home,file=".RData")
+			loc = File.join(loc,file)
+			connection.eval "load(\"#{loc}\")"
+		end
+
+		def get(connection, instruction)
+			connection.eval instruction
+		end
+
+		def get_vars(connection)
+			connection.eval("ls()")
+		end
+
+	end 
+
+	class Client
+		include R2RDF::Rconnect
+    attr :R
+    
+		def initialize(auto=true, loc=Dir.home)
+      @R = connect
+			@loc = loc
+			load_ws if auto
+			puts "vars: #{vars.payload}" if auto
+		end
+
+		def load_ws
+			load_workspace(@R, @loc)
+		end
+
+		def get_var(var)
+			get(@R,var)
+		end
+
+		def get_ws
+			"#{@loc}/.RData"
+		end
+
+		def vars
+			get_vars(@R)
+		end
+  end
+end

data/lib/bio-publisci/readers/arff.rb

@@ -0,0 +1,87 @@
+module R2RDF
+		module Reader
+		class ARFF
+			include R2RDF::Dataset::DataCube
+			def generate_n3(arff, options={})
+				arff = IO.read(arff) if File.exist? arff	
+				options[:no_labels] = true # unless options[:no_labels] == nil
+				@options = options
+				comps =  components(arff)
+				obs = data(arff, comps.keys)
+				repl = generate(comps.reject{|c| comps[c][:codes]}.keys, comps.select{|c| comps[c][:codes]}.keys, comps.select{|c| comps[c][:codes]}.keys, obs, (1..obs.first[1].size).to_a, relation(arff), options)
+				
+			end
+
+			def relation(arff)
+				arff.match(/@relation.+/i).to_a.first.split.last
+			end
+
+			def components(arff)
+				#still needs support for quoted strings with whitespace
+				h ={}
+				arff.split("\n").select{|lin| lin =~ /^@ATTRIBUTE/i}.map{|line|
+					if line =~ /\{.*}/
+						name = line.match(/\s.*/).to_a.first.strip.split.first
+						type = :coded
+						codes = line.match(/\{.*}/).to_a.first[1..-2].split(',')
+						h[name] = {type: type, codes: codes}
+					else
+						name = line.split[1]
+						type = line.split[2]
+						h[name] = {type: type}
+					end
+				}
+				h
+			end
+
+			def data(arff, attributes)
+				lines = arff.split("\n")
+				data_lines = lines[lines.index(lines.select{|line| line =~ /^@DATA/i}.first)+1..-1]
+				h=attributes.inject({}){|ha,attrib| ha[attrib] = []; ha}
+				data_lines.map{|line|
+					line = line.split ','
+					attributes.each_with_index{|a,i| h[a] << line[i]}
+				}
+				h
+			end
+
+		# 	def coded_dimensions
+		# 		if @options[:codes]
+		# 			@options[:codes]
+		# 		elsif @options[:row_label]
+		# 			[@options[:row_label]]
+		# 		else
+		# 			["refRow"]
+		# 		end	
+		# 	end
+
+		# 	def measures
+		# 		if @options[:dimensions]
+		# 			if @options[:measures]
+		# 				@options[:measures] - @options[:dimensions]
+		# 			else
+		# 				# @rexp.payload.names - @options[:dimensions]
+		# 			end
+		# 		else
+		# 			@options[:measures] # || @rexp.payload.names
+		# 		end
+		# 	end
+
+		# 	def observation_labels
+		# 		# row_names = @rexp.attr.payload["row.names"].to_ruby
+	 #   #    row_names = (1..@rexp.payload.first.to_ruby.size).to_a unless row_names.first
+	 #   #    row_names
+		# 	end
+
+		# 	def observation_data
+
+		# 		# data = {}
+		# 		# @rexp.payload.names.map{|name|
+		# 		# 	data[name] = @rexp.payload[name].to_ruby
+		# 		# }
+		# 		# data[@options[:row_label] || "refRow"] = observation_labels()
+		# 		# data
+		# 	end
+		end
+	end
+end

data/lib/bio-publisci/readers/big_cross.rb

@@ -0,0 +1,119 @@
+module R2RDF
+	module Reader
+		class BigCross
+			include R2RDF::Dataset::DataCube
+
+			def generate_n3(client, var, outfile_base, options={})
+				meas = measures(client,var,options)
+				dim = dimensions(client,var,options)
+				codes = codes(client,var,options)
+				
+
+				#write structure
+				open(outfile_base+'_structure.ttl','w'){|f| f.write structure(client,var,options)}
+				
+				n_individuals = client.eval("length(#{var}$pheno[[1]])").payload.first
+				chromosome_list = (1..19).to_a.map(&:to_s) + ["X"]
+				chromosome_list.map{|chrom|
+					open(outfile_base+"_#{chrom}.ttl",'w'){|f| f.write prefixes(var,options)}
+					entries_per_individual = client.eval("length(#{var}$geno$'#{chrom}'$map)").to_ruby
+
+					#get genotype data (currently only for chromosome 1)
+					# => puts "#{var}$geno$'#{chrom}'"
+					geno_chr = client.eval("#{var}$geno$'#{chrom}'")
+
+					#get number of markers per individual
+
+					#write observations
+					n_individuals.times{|indi|
+						#time ||= Time.now
+						obs_data = observation_data(client,var,chrom.to_s,indi,geno_chr,entries_per_individual,options)
+						labels = labels_for(obs_data,chrom.to_s,indi)
+						open(outfile_base+"_#{chrom}.ttl",'a'){|f| observations(meas,dim,codes,obs_data,labels,var,options).map{|obs| f.write obs}}
+						puts "(#{chrom}) #{indi}/#{n_individuals}" #(#{Time.now - time})
+						#time = Time.now
+					}
+				}
+
+				#generate(measures, dimensions, codes, observation_data, observation_labels, var, options)
+			end
+
+			def structure(client,var,options={})
+				meas = measures(client,var,options)
+				dim = dimensions(client,var,options)
+				codes = codes(client,var,options)
+
+				str = prefixes(var,options)
+				str << data_structure_definition(meas,var,options)
+				str << dataset(var,options)
+    		component_specifications(meas, dim, var, options).map{ |c| str << c }
+				measure_properties(meas,var,options).map{|m| str << m}
+				
+				str
+			end
+
+			def measures(client, var, options={})
+				pheno_names = client.eval("names(#{var}$pheno)").to_ruby
+				if options[:measures]
+					(pheno_names & options[:measures]) | ["genotype","markerpos","marker"]
+				else
+					pheno_names | ["genotype","markerpos","marker"]
+				end
+				# measure_properties(measures,var,options)
+			end
+
+			def dimensions(client, var, options={})
+				# dimension_properties([""],var)
+				[]
+			end
+
+			def codes(client, var, options={})
+				[]				
+			end
+
+			def labels_for(data,chr,individual,options={})
+				labels=(((data.first.last.size*individual)+1)..(data.first.last.size*(individual+1))).to_a.map(&:to_s)
+				labels.map{|l| l.insert(0,"#{chr}_")}
+				labels
+			end
+
+			def observation_data(client, var, chr, row_individ, geno_chr, entries_per_individual, options={})
+				data = {}
+				# geno_chr = client.eval("#{var}$geno$'#{chr}'")
+				# n_individuals = client.eval("#{var}$pheno[[1]]").to_ruby.size
+				# entries_per_individual = @rexp.payload["geno"].payload[row_individ].payload["map"].payload.size * @rexp.payload["geno"].payload.names.size
+				data["chr"] = []
+				data["genotype"] = []
+				data["individual"] = []
+				data["marker"] = []
+				data["markerpos"] = []
+				client.eval("names(#{var}$pheno)").to_ruby.map{|name|
+					data[name] = []
+				}
+				# n_individuals.times{|row_individ|
+					# puts "#{row_individ}/#{n_individuals}"
+				data["individual"] << (1..entries_per_individual).to_a.fill(row_individ)
+
+				client.eval("names(#{var}$pheno)").to_ruby.map{|name|
+					data[name] << (1..entries_per_individual).to_a.fill(client.eval("#{var}$pheno").payload[name].to_ruby[row_individ])
+				}
+				# @rexp.payload["geno"].payload.names.map { |chr|
+				num_markers = geno_chr.payload.first.to_ruby.column_size
+				data["chr"] << (1..num_markers).to_a.fill(chr)
+				data["genotype"] << geno_chr.payload["data"].to_ruby.row(row_individ).to_a
+				data["marker"] << client.eval("names(#{var}$geno$'#{chr}'$map)").payload
+				data["markerpos"] << geno_chr.payload["map"].to_a
+					# }
+				# }
+				data.map{|k,v| v.flatten!}
+				data
+			end
+
+			def num_individuals(client, var, options={})
+				client.eval("#{var}$pheno").payload.first.to_ruby.size
+			end
+
+
+		end
+	end
+end

data/lib/bio-publisci/readers/cross.rb

@@ -0,0 +1,72 @@
+module R2RDF
+	module Reader
+		class Cross
+			include R2RDF::Dataset::DataCube
+
+			def generate_n3(rexp, var, options={})
+				@rexp = rexp
+				@options = options
+				generate(measures, dimensions, codes, observation_data, observation_labels, var, options)
+			end
+
+			def dimensions
+				["individual","chr","sex","marker"]
+			end
+
+			def codes
+				["individual","chr","sex","marker"]
+			end
+
+			def measures
+				((@rexp.payload["pheno"].payload.names - ["sex"]) | ["genotype","markerpos"]) 
+			end
+
+			def observation_labels
+				# row_names = @rexp.attr.payload["row.names"].to_ruby
+				# entries_per_individual = @rexp.payload["geno"].payload[0].payload["map"].payload.size * @rexp.payload["geno"].payload.names.size
+				entries_per_individual = 0
+				@rexp.payload["geno"].payload.to_ruby.map{|v| entries_per_individual += (v["map"].size)}
+				individuals = @rexp.payload["pheno"].payload.first.to_ruby.size
+	      (1..(entries_per_individual * individuals)).to_a
+			end
+
+			def observation_data
+
+				data = {}
+				n_individuals = @rexp.payload["pheno"].payload.first.to_ruby.size
+				entries_per_individual = 0
+				@rexp.payload["geno"].payload.to_ruby.map{|v| entries_per_individual += (v["map"].size)}
+				# entries_per_individual = @rexp.payload["geno"].payload[row_individ].payload["map"].payload.size * @rexp.payload["geno"].payload.names.size
+				data["chr"] = []
+				data["genotype"] = []
+				data["individual"] = []
+				data["marker"] = []
+				data["markerpos"] = []
+				@rexp.payload["pheno"].payload.names.map{|name|
+					data[name] = []
+				}
+				n_individuals.times{|row_individ|
+					# puts row_individ
+					data["individual"] << (1..entries_per_individual).to_a.fill(row_individ)
+					@rexp.payload["pheno"].payload.names.map{|name|
+						data[name] << (1..entries_per_individual).to_a.fill(@rexp.payload["pheno"].payload[name].to_ruby[row_individ])
+					}
+					@rexp.payload["geno"].payload.names.map { |chr|
+						geno_chr = @rexp.payload["geno"].payload[chr]
+						num_markers = geno_chr.payload.first.to_ruby.column_size
+						data["chr"] << (1..num_markers).to_a.fill(chr)
+						data["genotype"] << geno_chr.payload["data"].to_ruby.row(row_individ).to_a
+						data["marker"] << geno_chr.payload["map"].to_ruby.names
+						data["markerpos"] << geno_chr.payload["map"].to_a
+					}
+				}
+				# data["chr"].flatten!
+				# data["genotype"].flatten!
+				data.keys.map{|k| data[k].flatten!}
+
+				#data["refRow"] = observation_labels()
+				data
+			end
+		end
+	end
+end