bio-publisci 0.0.1

data/lib/bio-publisci/readers/csv.rb

@@ -0,0 +1,54 @@
+module R2RDF
+	module Reader
+		class CSV
+			include R2RDF::Dataset::DataCube
+
+			def generate_n3(file, dataset_name, options={})
+				@data = ::CSV.read(file)
+				@options = options
+				generate(measures, dimensions, codes, observation_data, observation_labels, dataset_name, options)
+			end
+
+			def dimensions
+				@options[:dimensions] || [@data[0][0]]
+			end
+
+			def codes
+				@options[:codes] || dimensions()
+			end
+
+			def measures
+				@options[:measures] || @data[0] - dimensions()
+			end
+
+			def observation_labels
+				if @options[:label_column]
+					tmp = @data.dup
+					tmp.shift
+					tmp.map{|row|
+						row[@options[:label_column]]
+					}
+				else
+					(1..@data.size - 1).to_a
+				end
+			end
+
+			def observation_data
+
+				obs = {}
+				@data[0].map{|label|
+					obs[label] = []
+				}
+				tmp = @data.dup
+				tmp.shift
+				
+				tmp.map{|row|
+					row.each_with_index{|entry,i|
+						obs[@data[0][i]] << entry
+					}
+				}
+				obs
+			end
+		end
+	end
+end

data/lib/bio-publisci/readers/dataframe.rb

@@ -0,0 +1,66 @@
+module R2RDF
+		module Reader
+		class Dataframe
+			include R2RDF::Dataset::DataCube
+		
+			# def initialize(var)
+			# 	@var = var
+			# end
+			
+			def generate_n3(rexp, var, options={})
+				@rexp = rexp
+				@options = options
+
+				generate(measures, dimensions, codes, observation_data, observation_labels, var, options)
+			end
+
+			def dimensions
+				if @options[:dimensions]
+					@options[:dimensions]
+				elsif @options[:row_label]
+					[@options[:row_label]]
+				else
+					["refRow"]
+				end	
+			end
+
+			def codes
+				if @options[:codes]
+					@options[:codes]
+				elsif @options[:row_label]
+					[@options[:row_label]]
+				else
+					["refRow"]
+				end	
+			end
+
+			def measures
+				if @options[:dimensions]
+					if @options[:measures]
+						@options[:measures] - @options[:dimensions]
+					else
+						@rexp.payload.names - @options[:dimensions]
+					end
+				else
+					@options[:measures] || @rexp.payload.names
+				end
+			end
+
+			def observation_labels
+				row_names = @rexp.attr.payload["row.names"].to_ruby
+	      row_names = (1..@rexp.payload.first.to_ruby.size).to_a unless row_names.first
+	      row_names
+			end
+
+			def observation_data
+
+				data = {}
+				@rexp.payload.names.map{|name|
+					data[name] = @rexp.payload[name].to_ruby
+				}
+				data[@options[:row_label] || "refRow"] = observation_labels()
+				data
+			end
+		end
+	end
+end

data/lib/bio-publisci/readers/r_matrix.rb

@@ -0,0 +1,152 @@
+module R2RDF
+	module Reader
+		class RMatrix
+			include R2RDF::Dataset::DataCube
+
+			#NOTE; this is pretty much hard coded for Karl's application right now, and doesn't
+			# do any dimension or code generation. Since its a set of LOD scores indexed by dimension
+			# and marker the usual datacube generator wont work (I think). In the future adding an option
+			# to specify this kind of a dataset would probably be useful
+
+
+			def generate_n3(client, var, outfile_base, options={})
+				meas = measures(client,var,options)
+				dim = dimensions(client,var,options)
+				codes = codes(client,var,options)
+				
+				outvar = sanitize([var]).first
+				
+				probes_per_file = options[:probes_per_file] || 100
+				col_select = "colnames" 
+				col_select = "names" if options[:type] == :dataframe
+
+				#write structure
+				open(outfile_base+'_structure.ttl','w'){|f| f.write structure(client,var,outvar,options)}
+
+				probes=client.eval("#{col_select}(#{var})").to_ruby
+				if probes == nil
+					client.eval("colnames(#{var})=1:ncol(#{var})")
+					probes=client.eval("#{col_select}(#{var})").to_ruby
+				end
+				markers = rows(client,var,options)
+
+				probes.each_with_index{|probe,i|
+					#write prefixes and erase old file on first run
+					open(outfile_base+"_#{i/probes_per_file}.ttl",'w'){|f| f.write prefixes(var,options)} if i % probes_per_file == 0
+					i+=1
+					obs_data = observation_data(client,var,i,markers,options)
+					labels = labels_for(client,var,probe)
+					
+					# labels = sanitize(labels)
+					# return obs_data
+					open(outfile_base+"_#{i/probes_per_file}.ttl",'a'){|f| observations(meas,dim,codes,obs_data,labels,outvar,options).map{|obs| f.write obs}}
+					puts "#{i}/#{probes.size}" unless options[:quiet]
+				}
+			end
+
+			def structure(client,var,outvar,options={})
+				meas = measures(client,var,options)
+				dim = dimensions(client,var,options)
+				codes = codes(client,var,options)
+
+				str = prefixes(var, options)
+				str << data_structure_definition(meas,outvar,options)
+				str << dataset(outvar,options)
+    		component_specifications(meas, dim, var, options).map{ |c| str << c }
+				measure_properties(meas,var,options).map{|m| str << m}
+				
+				str
+			end
+
+			#for now just make everything a measure
+			def measures(client, var, options={})
+				if options[:measures]
+						options[:measures] 
+				else
+					["probe","marker","value"]
+				end
+				# measure_properties(measures,var,options)
+			end
+
+			def dimensions(client, var, options={})
+				# dimension_properties([""],var)
+				[]
+			end
+
+			def codes(client, var, options={})
+				[]				
+			end
+
+			def labels_for(connection,var,probe_id,options={})
+				row_names = connection.eval("row.names(#{var})")
+				# row_names = (1..@rexp.payload.first.to_ruby.size).to_a unless row_names.first
+				if row_names == connection.eval('NULL')
+					row_names = (1..connection.eval("nrow(#{var})").payload.first).to_a
+				else
+					row_names = row_names.payload
+				end
+
+	      labels = (1..(row_names.size)).to_a.map(&:to_s)
+	      labels = labels.map{|l|
+	      	l.insert(0,probe_id.to_s + "_")
+	      }
+
+	      labels
+			end
+
+			def rows(connection,var,options={})
+				row_names = connection.eval("row.names(#{var})")
+				#hacky solution because rserve client's .to_ruby method doesn't fully work
+				if row_names == connection.eval('NULL')
+					row_names = (1..connection.eval("nrow(#{var})").payload.first).to_a
+				else
+					row_names = row_names.payload
+				end
+	      row_names
+			end
+
+			def observation_data(client, var, probe_number, row_names, options={})
+
+				data = {}
+				# geno_chr = client.eval("#{var}$geno$'#{chr}'")
+				# n_individuals = client.eval("#{var}$pheno[[1]]").to_ruby.size
+				# entries_per_individual = @rexp.payload["geno"].payload[row_individ].payload["map"].payload.size * @rexp.payload["geno"].payload.names.size
+				col_label = "probe"
+				row_label = "marker"
+				val_label = "value"
+
+				if options[:measures]
+					col_label = options[:measures][0] || "probe"
+					row_label = options[:measures][1] || "marker"
+					val_label = options[:measures][2] || "value"
+				end
+
+				data["#{col_label}"] = []
+				data["#{row_label}"] = []
+				data["#{val_label}"] = []
+				
+				# n_individuals.times{|row_individ|
+					# puts "#{row_individ}/#{n_individuals}"
+
+				col_select = "colnames" 
+				col_select = "names" if options[:type] == :dataframe
+
+				if options[:type] == :dataframe
+					probe_obj = client.eval("#{var}[[#{probe_number}]]").to_ruby
+				else
+					probe_obj = client.eval("#{var}[,#{probe_number}]").to_ruby
+				end
+				# puts probe_obj
+				probe_id = client.eval("#{col_select}(#{var})[[#{probe_number}]]").to_ruby
+				data["#{col_label}"] = (1..(probe_obj.size)).to_a.fill(probe_id)
+				probe_obj.each_with_index{|lod,i|
+					data["#{row_label}"] << row_names[i]
+					data["#{val_label}"] << lod
+				}
+				
+				data.map{|k,v| v.flatten!}
+				data
+			end
+		end
+	end
+end

data/lib/bio-publisci/store.rb

@@ -0,0 +1,56 @@
+module R2RDF
+  # handles connection and messaging to/from the triple store
+  class Store
+  	include R2RDF::Query
+
+    def defaults
+	    {
+	      type: :fourstore,
+	      url: "http://localhost:8080", #TODO port etc should eventually be extracted from URI if given
+        replace: false
+	    }
+	  end
+
+	  def add(file,graph)
+	  	if @options[:type] == :graph
+	  		throw "please provide an RDF::Repository" unless graph.is_a? RDF::Repository
+	  		graph.load(file)
+	  		@store = graph
+	  		@store
+	  	elsif @options[:type] == :fourstore
+		  	if @options[:replace]
+          `curl -T #{file} -H 'Content-Type: application/x-turtle' #{@options[:url]}/data/http%3A%2F%2Frqtl.org%2F#{graph}`
+        else
+          `curl --data-urlencode data@#{file} -d 'graph=http%3A%2F%2Frqtl.org%2F#{graph}' -d 'mime-type=application/x-turtle' #{@options[:url]}/data/`
+        end
+		  end
+	  end
+
+	  def add_all(dir, graph, pattern=nil)
+	  	pattern = /.+\.ttl/ if pattern == :turtle || pattern == :ttl 
+
+	  	files = Dir.entries(dir) - %w(. ..)
+	  	files = files.grep(pattern) if pattern.is_a? Regexp
+	  	nfiles = files.size
+			n = 0
+			files.each{|file| puts file + " #{n+=1}/#{nfiles} files"; puts add(file,graph)}
+	  end
+
+    def initialize(options={})
+      @options = defaults.merge(options)
+    end
+
+    def query(string)
+    	# execute(string, )
+			if @options[:type] == :graph
+				execute(string, @store, :graph)
+			elsif @options[:type] == :fourstore
+				execute(string, @options[:url], :fourstore)
+		  end
+    end
+
+    def url
+    	@options[:url]
+    end
+  end
+end

data/lib/bio-publisci/writers/arff.rb

@@ -0,0 +1,66 @@
+module R2RDF
+	module Writer
+		class ARFF
+			include R2RDF::Query
+			include R2RDF::Parser
+			include R2RDF::Analyzer
+
+			def build_arff(relation, attributes, data, source)
+				str = <<-EOS
+% 1. Title: #{relation.capitalize} Database
+%
+% 2. Sources:
+%    (a) Generated from RDF source #{source}
+%     
+@RELATION #{relation}
+
+EOS
+
+				Hash[attributes.sort].map{|attribute,type|
+					str << "@ATTRIBUTE #{attribute} #{type}\n"
+				}
+
+				str << "\n@DATA\n"
+				data.map { |d| str << Hash[d[1].sort].values.join(',') + "\n" }
+
+				str
+			end
+
+			def from_turtle(turtle_file, verbose=false)
+				puts "loading #{turtle_file}" if verbose
+				repo = RDF::Repository.load(turtle_file)
+				puts "loaded #{repo.size} statements into temporary repo" if verbose
+				
+				dims = get_ary(execute_from_file("dimensions.rq",repo,:graph)).flatten
+				meas = get_ary(execute_from_file("measures.rq",repo,:graph)).flatten
+				relation = execute_from_file("dataset.rq",repo,:graph).to_h.first[:label].to_s
+				codes = execute_from_file("codes.rq",repo,:graph).to_h.map{|e| e.values.map(&:to_s)}.inject({}){|h,el|
+					(h[el.first]||=[]) << el.last; h
+				}
+
+				data = observation_hash(execute_from_file("observations.rq",repo,:graph), true)
+				attributes = {}
+				(dims | meas).map{|component|
+					attributes[component] = case recommend_range(data.map{|o| o[1][component]})
+						when "xsd:int"
+							"integer"
+						when "xsd:double"
+							"real"
+						when :coded
+							if dims.include? component
+								"{#{codes[component].join(',')}}"
+							else
+								"string"
+							end
+						end
+				}
+
+				build_arff(relation, attributes, data, turtle_file)
+			end
+
+			def from_store(endpoint_url,variable_in=nil, variable_out=nil, verbose=false)
+				raise "not implemented yet"
+			end
+		end
+	end
+end

data/lib/bio-publisci/writers/dataframe.rb

@@ -0,0 +1,81 @@
+module R2RDF
+		module Writer
+	  	module Dataframe
+
+	    def framestring(name,vectors)
+	      framestr = "#{name} = data.frame("
+	      vectors.map{ |k,v| framestr << k + '=' + k +','}
+	      framestr[-1] = ')'
+	      framestr
+	    end
+	    
+	    def get_vectors(variable_name, helper, repo)
+	      column_names = helper.get_ary(helper.execute(helper.property_names(variable_name), repo)).flatten.map{|n| n.gsub(' Component','')}
+	      vectors = {}
+	      column_names.map{|n| 
+	        vectors[n] = helper.get_ary(helper.execute(helper.property_values(variable_name,n),repo),'to_f').flatten unless n == "refRow"
+	      }
+	      vectors
+	    end
+
+	    def create_dataframe(name, connection, rows, vectors)
+	      connection.assign('rows', rows)
+	      vectors.map{ |k,v| 
+	        connection.assign(k,v) 
+	      }
+	      connection.eval(framestring(name,vectors))
+	      connection.eval("row.names(#{name}) <- rows")
+	      connection.eval(name)
+	    end
+
+	    def save_workspace(connection, loc)
+	    	connection.eval "save.image(#{loc})"
+	    end
+
+	    def get_rownames(variable, helper, repo)
+	      rows = helper.get_ary(helper.execute(helper.row_names(variable), repo)).flatten
+	    end
+
+	  end
+
+	  class Builder
+	    include R2RDF::Writer::Dataframe
+
+
+	    def from_turtle(turtle_file, connection, variable_in=nil, variable_out=nil, verbose=true, save=true)
+	      unless variable_in && variable_out
+	        puts "no variable specified. Simple inference coming soon" if verbose
+	        return
+	      end
+	      puts "loading #{turtle_file}" if verbose
+	      repo = RDF::Repository.load(turtle_file)
+	      puts "loaded #{repo.size} statements into temporary repo" if verbose
+	      # connection = Rserve::Connection.new
+	      query = R2RDF::QueryHelper.new
+	      rows = get_rownames(variable_in, query, repo)
+	      puts "frame has #{rows.size} rows" if verbose
+
+	      vectors = get_vectors(variable_in, query, repo)
+	      puts "got vectors of size #{vectors.first.last.size}" if verbose && vectors.first
+
+	      create_dataframe(variable_out, connection, rows, vectors)
+	      save_workspace(connection, connection.eval('getwd()').to_ruby) if save
+	    end
+
+	    def from_store(endpoint_url,connection,variable_in=nil, variable_out=nil, verbose=true, save=true)
+	    	unless variable_in && variable_out
+	    	  puts "no variable specified. Simple inference coming soon" if verbose
+	    	  return
+	    	end
+	    	puts "connecting to endpoint at #{endpoint_url}" if verbose
+	    	sparql = SPARQL::Client.new(endpoint_url)
+	    	# client = R2RDF::Client.new
+	      query = R2RDF::QueryHelper.new
+
+	      rows = query.get_ary(sparql.query(query.row_names(variable_in))).flatten
+
+	    end
+
+	  end
+	end
+end

data/lib/bio-publisci.rb

@@ -0,0 +1,36 @@
+# This is temporary, just to help w/ development so I don't have to rewrite r2rdf.rb to be
+# a standard gem base yet. Also load s the files instead of require for easy reloading
+require 'tempfile'
+require 'rdf'
+require 'csv'
+require 'rserve'
+require 'sparql'
+require 'sparql/client'
+require 'rdf/turtle'
+
+def load_folder(folder)
+	Dir.foreach(File.dirname(__FILE__) + "/#{folder}") do |file|
+		unless file == "." or file == ".."
+			load File.dirname(__FILE__) + "/#{folder}/" + file
+		end
+	end
+end
+
+load File.dirname(__FILE__) + '/bio-publisci/dataset/interactive.rb'
+load File.dirname(__FILE__) + '/bio-publisci/query/query_helper.rb'
+load File.dirname(__FILE__) + '/bio-publisci/parser.rb'
+load File.dirname(__FILE__) + '/bio-publisci/r_client.rb'
+load File.dirname(__FILE__) + '/bio-publisci/analyzer.rb'
+load File.dirname(__FILE__) + '/bio-publisci/store.rb'
+load File.dirname(__FILE__) + '/bio-publisci/dataset/data_cube.rb'
+
+
+load_folder('bio-publisci/metadata')
+load_folder('bio-publisci/readers')
+load_folder('bio-publisci/writers')
+load_folder('bio-publisci/dataset/ORM')
+# Dir.foreach(File.dirname(__FILE__) + '/generators') do |file|
+# 	unless file == "." or file == ".."
+# 		load File.dirname(__FILE__) + '/generators/' + file
+# 	end
+# end