bio-publisci 0.0.2 → 0.0.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f03476f5595b28e10a4cb3d950b9c454d621eb4d
+  data.tar.gz: da473d84d9c1e203de4ea1b2d7fef2579cfa8bf8
+SHA512:
+  metadata.gz: 33662112c4df2115f15d59f415329dfa11368e8bd1a47907d62c02b476bbeb4afe0a1c43599d2c215a093d1765b7ffb05320220e21888267811c86beacef8876
+  data.tar.gz: 683b5874d082ab155227ee92a1e9bfe15a66b0eaf4fe1e590db39a9391346ec31850b0a94d924c9d26b356fb6ed7f82a228c9d16afa1a67d9c6710086e849963

data/.travis.yml

@@ -1,7 +1,7 @@
 language: ruby
 rvm:
-  - 1.9.2
   - 1.9.3
+  - 2.0.0
   - jruby-19mode # JRuby in 1.9 mode
 
 # - rbx-19mode

data/Gemfile

@@ -17,7 +17,7 @@ end
 
 
 gem 'rdf'
-gem 'rdf-4store'
+gem 'rdf-4store', :git => "https://github.com/fumi/rdf-4store.git"
 gem 'rdf-turtle'
 gem 'rserve-client'
 gem 'sparql'

data/Rakefile

@@ -21,7 +21,8 @@ Jeweler::Tasks.new do |gem|
   gem.description = %Q{A toolkit for publishing scientific results and datasets using RDF and related technologies }
   gem.email = "wstrinz@gmail.com"
   gem.authors = ["wstrinz"]
-  gem.version = "0.0.2"
+  gem.version = "0.0.3"
+
   # dependencies defined in Gemfile
 end
 Jeweler::RubygemsDotOrgTasks.new
@@ -29,6 +30,7 @@ Jeweler::RubygemsDotOrgTasks.new
 require 'rspec/core'
 require 'rspec/core/rake_task'
 RSpec::Core::RakeTask.new(:spec) do |spec|
+  spec.rspec_opts = "--tag ~no_travis"
   spec.pattern = FileList['spec/**/*_spec.rb']
 end
 
@@ -44,11 +46,7 @@ Cucumber::Rake::Task.new(:features)
 # task :default => :spec
 
 task :default => [] do
-  begin   
-    Rake::Task[:spec].invoke
-  rescue
-  end
-  Rake::Task[:features].invoke
+  Rake::Task[:spec].invoke
 end
 
 task :test => [] do

data/features/integration_steps.rb

@@ -1,5 +1,5 @@
+
 Then(/^I should be able to cluster the result and print statistics$/) do
-  require 'ruby_mining'
   f=Tempfile.open('arff'); f.write @result; f.close
   clustering = Weka::Clusterer::SimpleKMeans::Base
   clustering.set_options "-N 5"

data/features/metadata.feature

@@ -0,0 +1,24 @@
+Feature: Receive metadata as user input or extract from data sources
+
+  In order to publish and share data about my datasets
+  I want to be able to attach metadata
+
+  Scenario: Attach basic DC Terms info
+    Given a class which includes the Metadata module
+    When I call its basic method with the hash {var: "example", title: "example dataset", creator: "Will Strinz", description: "an example dataset", date: "1-10-2010"}
+    Then I should receive a metadata string
+
+  Scenario: Auto Generate some fields
+    Given a class which includes the Metadata module
+    When I call its basic method with the hash {var: "example", title: "example dataset", description: "an example dataset"}
+    Then I should receive a metadata string
+
+  Scenario: Generate process information
+    Given a class which includes the Metadata module
+    When I call its provenance method with the hash {var: "example", software: {name: "R", process: 'spec/resource/example.Rhistory'}}
+    Then I should receive a metadata string
+
+  Scenario: Generate organizational provenance information
+    Given a class which includes the Metadata module
+    When I call its provenance method with the hash {var: "example", creator: "http://gsocsemantic.wordpress.com/me", organization: "http://sciruby.com/"}
+    Then I should receive a metadata string

data/features/metadata_steps.rb

@@ -0,0 +1,21 @@
+Given(/^a class which includes the Metadata module$/) do
+  class Meta
+    include R2RDF::Metadata
+  end
+  @klass = Meta
+end
+
+When(/^I call its basic method with the hash (\{.+\})$/) do |fields|
+  fields = eval(fields)
+  @response = @klass.new.basic(fields)
+end
+
+When(/^I call its provenance method with the hash (\{.+\})$/) do |fields|
+  fields = eval(fields)
+  @response = @klass.new.provenance(fields)
+end
+
+Then(/^I should receive a metadata string$/) do
+  @response.is_a?(String).should be true
+  puts @response
+end

data/lib/bio-publisci/dataset/ORM/data_cube_orm.rb

@@ -1,5 +1,5 @@
 module R2RDF
-	module Dataset
+	class Dataset
 		module ORM
 			class DataCube
 				extend R2RDF::Dataset::DataCube
@@ -43,25 +43,41 @@ module R2RDF
 						if solution[:range].split('/')[-2] == "code"
 							type = :coded
 						else
-							type = strip_uri(solution[:range])
+							type = solution[:range].to_s
 						end
-						[strip_uri(solution[:dimension]), {type: type}]
+						[solution[:dimension], {type: type}]
 					}]
 					puts "dimensions: #{dimensions}" if verbose
-					measures = get_ary(execute_from_file('measures.rq',graph)).flatten
-					puts "measures: #{measures}" if verbose
+
+          codes = execute_from_file('code_resources.rq',graph).to_h.map{|sol|
+            [sol[:dimension].to_s, sol[:codeList].to_s, sol[:class].to_s]
+          }
+					puts "codes: #{codes}" if verbose
+          
+          measures = execute_from_file('measures.rq',graph).to_h.map{|m| m[:measure].to_s}
+          puts "measures: #{measures}" if verbose
+          
           name = execute_from_file('dataset.rq',graph).to_h.first[:label]
           puts "dataset: #{name}" if verbose
-					obs = execute_from_file('observations.rq',graph)
-          puts "observations: #{obs}" if verbose
-          # observations = observation_hash(obs)
-          simple_observations = observation_hash(obs,true)
+          
+          obs = execute_from_file('observations.rq',graph)
+          observations = observation_hash(obs)
+          puts "observations: #{observations}" if verbose
+
+          # simple_observations = observation_hash(obs,true)
+
+          labels = execute_from_file('observation_labels.rq', graph)
+          labels = Hash[labels.map{|sol|
+            [sol[:observation].to_s, sol[:label].to_s]
+          }]
 
 					new_opts = {
 						measures: measures,
 						dimensions: dimensions,
-						observations: simple_observations.values,
+						observations: observations.values,
 						name: name,
+            labels: labels.values,
+            codes: codes
 					}
 
 					options = options.merge(new_opts)
@@ -96,6 +112,14 @@ module R2RDF
 					if options[:validate_each]
 						@options[:validate_each] = options[:validate_each]
 					end
+
+          if options[:labels]
+            @labels = options[:labels]
+          end
+
+          if options[:codes]
+            @codes = options[:codes]
+          end
 				end
 
 				def to_n3
@@ -120,10 +144,8 @@ module R2RDF
 					}
 					
 
-					codes = @dimensions.map{|d,v| d if v[:type] == :coded}.compact
-
-
-					str = generate(@measures, @dimensions.keys, codes, data, @labels, @name, @generator_options)
+					@codes = @dimensions.map{|d,v| d if v[:type] == :coded}.compact unless @codes
+					str = generate(@measures, @dimensions.keys, @codes, data, @labels, @name, @generator_options)
 					unless @options[:skip_metadata]
 		        fields = {
 		          publishers: publishers(),

data/lib/bio-publisci/dataset/ORM/observation.rb

@@ -1,5 +1,5 @@
 module R2RDF
-	module Dataset
+	class Dataset
 		module ORM
 			class Observation
 				attr_accessor :data			

data/lib/bio-publisci/dataset/data_cube.rb

@@ -1,68 +1,112 @@
   #monkey patch to make rdf string w/ heredocs prettier ;)  
-  class String
-    def unindent
-      gsub /^#{self[/\A\s*/]}/, ''
-     # gsub(/^#{scan(/^\s*/).min_by{|l|l.length}}/, "")
-    end
+class String
+  def unindent
+    gsub /^#{self[/\A\s*/]}/, ''
   end
+end
 
 module R2RDF
-  # used to generate data cube observations, data structure definitions, etc
-  module Dataset
+  class Dataset
     module DataCube
+      include R2RDF::Parser
       def defaults
       {
         type: :dataframe,
         encode_nulls: false,
         base_url: "http://www.rqtl.org",
       }
-     end
-     
-     def generate(measures, dimensions, codes, data, observation_labels, var, options={})
-      dimensions = sanitize(dimensions)
-      codes = sanitize(codes)
-      measures = sanitize(measures)
-      var = sanitize([var]).first
-      data = sanitize_hash(data)
-
-      str = prefixes(var,options)
-      str << data_structure_definition((measures | dimensions), var, options)
-      str << dataset(var, options)
-      component_specifications(measures, dimensions, var, options).map{ |c| str << c }
-      dimension_properties(dimensions, codes, var, options).map{|p| str << p}
-      measure_properties(measures, var, options).map{|p| str << p}
-      code_lists(codes, data, var, options).map{|l| str << l}
-      concept_codes(codes, data, var, options).map{|c| str << c}
-      observations(measures, dimensions, codes, data, observation_labels, var, options).map{|o| str << o}
-      str
-    end
+      end
 
-      def sanitize(array)
-        #remove spaces and other special characters
-        processed = []
-        array.map{|entry|
-          if entry.is_a? String
-            processed << entry.gsub(/[\s\.]/,'_')
+      def generate_resources(measures, dimensions, codes, options={})
+        newm = measures.map {|m|
+          if m =~ /^http:\/\//
+            "<#{m}>"
+          elsif m =~ /^[a-zA-z]+:[a-zA-z]+$/
+            m
           else
-            processed << entry
-          end
+            "prop:#{m}"
+          end            
         }
-        processed
-      end
 
-      def sanitize_hash(h)
-        mappings = {}
-        h.keys.map{|k| 
-          if(k.is_a? String)
-            mappings[k] = k.gsub(' ','_')
-          end
+        newc = []
+
+        newd = dimensions.map{|d|
+            if d =~ /^http:\/\//
+              # newc << "<#{d}>" if codes.include? d
+              "<#{d}>"
+            elsif d =~ /^[a-zA-z]+:[a-zA-z]+$/
+              d
+            else
+              # newc << "prop:#{d}" if codes.include? d
+              "prop:#{d}"
+            end
         }
 
-        h.keys.map{|k|
-          h[mappings[k]] = h.delete(k) if mappings[k]
+        if codes.first.is_a? Array
+          newc = codes.map{|c|
+            c.map{|el|
+              if el =~ /^http:\/\//
+                "<#{el}>"
+              else
+                el
+              end
+            }
+          }
+        else
+          newc = codes.map{|c|
+              ["#{c}","code:#{c.downcase}","code:#{c.downcase.capitalize}"]
+          }
+        end
+        [newm, newd, newc]
+      end
+
+      def encode_data(codes,data,var,options={})
+        new_data = {}
+        data.map{|k,v|
+          if codes.include? k
+            new_data[k] = v.map{|val|
+              if val =~ /^http:\/\//
+                "<#{val}>"
+              elsif val =~ /^[a-zA-z]+:[a-zA-z]+$/
+                val
+              else
+                "<code/#{k.downcase}/#{val}>"
+              end
+            }
+          else
+            new_data[k] = v
+          end
         }
+        new_data
+      end
 
-        h
+      def vocabulary(vocab,options={})
+        if vocab.is_a?(String) && vocab =~ /^http:\/\//
+          RDF::Vocabulary.new(vocab)
+        elsif RDF.const_defined? vocab.to_sym && RDF.const_get(vocab.to_sym).inspect =~ /^RDF::Vocabulary/
+          RDF.const_get(vocab)
+        else
+          nil
+        end
+      end
+     
+      def generate(measures, dimensions, codes, data, observation_labels, var, options={})
+        # dimensions = sanitize(dimensions)
+        # codes = sanitize(codes)
+        # measures = sanitize(measures)
+        var = sanitize([var]).first
+        data = sanitize_hash(data)
+
+        str = prefixes(var,options)
+        str << data_structure_definition(measures, dimensions, codes, var, options)
+        str << dataset(var, options)
+        # component_specifications(measures, dimensions, var, options).map{ |c| str << c }
+        dimension_properties(dimensions, codes, var, options).map{|p| str << p}
+        measure_properties(measures, var, options).map{|p| str << p}
+        code_lists(codes, data, var, options).map{|l| str << l}
+        concept_codes(codes, data, var, options).map{|c| str << c}
+        observations(measures, dimensions, codes, data, observation_labels, var, options).map{|o| str << o}
+        str
       end
 
       def prefixes(var, options={})
@@ -80,7 +124,6 @@ module R2RDF
         @prefix xsd:   <http://www.w3.org/2001/XMLSchema#> .
         @prefix cs:    <#{base}/dc/dataset/#{var}/cs/> .
         @prefix code:  <#{base}/dc/dataset/#{var}/code/> .
-        @prefix class: <#{base}/dc/dataset/#{var}/class/> .
         @prefix owl:   <http://www.w3.org/2002/07/owl#> .
         @prefix skos:  <http://www.w3.org/2004/02/skos/core#> .
         @prefix foaf:     <http://xmlns.com/foaf/0.1/> .
@@ -90,13 +133,18 @@ module R2RDF
         EOF
       end
 
-      def data_structure_definition(components,var,options={})
+      def data_structure_definition(measures,dimensions,codes,var,options={})
         var = sanitize([var]).first
         options = defaults().merge(options)
+        rdf_measures, rdf_dimensions, rdf_codes  = generate_resources(measures, dimensions, codes, options)
+
         str = "ns:dsd-#{var} a qb:DataStructureDefinition;\n"
-        str << "  qb:component\n"
-        components.map{|n|
-              str << "    cs:#{n} ,\n"
+        rdf_dimensions.map{|d|
+          str << "  qb:component [ qb:dimension #{d} ] ;\n"
+        }
+
+        rdf_measures.map{|m|
+          str << "  qb:component [ qb:measure #{m} ] ;\n"
         }
         str[-2]='.'
         str<<"\n"
@@ -141,24 +189,34 @@ module R2RDF
 
       def dimension_properties(dimensions, codes, var, options={})
         options = defaults().merge(options)
+        rdf_measures, rdf_dimensions, rdf_codes  = generate_resources([], dimensions, codes, options)
         props = []
-        
-          dimensions.map{|d|  
-            if codes.include?(d)
-              props << <<-EOF.unindent
-              prop:#{d} a rdf:Property, qb:DimensionProperty ;
-                rdfs:label "#{d}"@en ;
-                qb:codeList code:#{d.downcase} ;
-                rdfs:range code:#{d.downcase.capitalize} .
 
-              EOF
-            else
-              props << <<-EOF.unindent
-              prop:#{d} a rdf:Property, qb:DimensionProperty ;
-                rdfs:label "#{d}"@en .
+        dimension_codes = rdf_codes.map{|c| 
+          if c[0]=~/^<http:/
+            c[0][1..-2] 
+          else 
+            c[0]
+          end
+        }
 
-              EOF
-            end
+        rdf_dimensions.each_with_index{|d,i|
+          if dimension_codes.include?(dimensions[i])
+            code = rdf_codes[dimension_codes.index(dimensions[i])]
+            props << <<-EOF.unindent
+            #{d} a rdf:Property, qb:DimensionProperty ;
+              rdfs:label "#{strip_prefixes(strip_uri(d))}"@en ;
+              qb:codeList #{code[1]} ;
+              rdfs:range #{code[2]} .
+
+            EOF
+          else
+            props << <<-EOF.unindent
+            #{d} a rdf:Property, qb:DimensionProperty ;
+              rdfs:label "#{strip_prefixes(strip_uri(d))}"@en .
+
+            EOF
+          end
           }
         
         props
@@ -166,13 +224,14 @@ module R2RDF
 
       def measure_properties(measures, var, options={})
         options = defaults().merge(options)
+        rdf_measures = generate_resources(measures, [], [], options)[0]
         props = []
         
-          measures.map{ |m|
+          rdf_measures.map{ |m|
               
             props <<  <<-EOF.unindent
-            prop:#{m} a rdf:Property, qb:MeasureProperty ;
-              rdfs:label "#{m}"@en .
+            #{m} a rdf:Property, qb:MeasureProperty ;
+              rdfs:label "#{strip_prefixes(strip_uri(m))}"@en .
 
             EOF
             }
@@ -183,7 +242,18 @@ module R2RDF
       def observations(measures, dimensions, codes, data, observation_labels, var, options={})  
         var = sanitize([var]).first
         options = defaults().merge(options)
+        rdf_measures, rdf_dimensions, rdf_codes  = generate_resources(measures, dimensions, codes, options)
+        data = encode_data(codes, data, var, options)
         obs = []
+        
+        dimension_codes = rdf_codes.map{|c| 
+          if c[0]=~/^<http:/
+            c[0][1..-2] 
+          else 
+            c[0]
+          end
+        }
+
         observation_labels.each_with_index.map{|r, i|
           contains_nulls = false
           str = <<-EOF.unindent 
@@ -193,48 +263,64 @@ module R2RDF
 
           str << "  rdfs:label \"#{r}\" ;\n" unless options[:no_labels]
           
-          dimensions.map{|d|
+          dimensions.each_with_index{|d,j|
             contains_nulls = contains_nulls | (data[d][i] == nil)
-            if codes.include? d
-              str << "  prop:#{d} <code/#{d.downcase}/#{data[d][i]}> ;\n"
+
+            if dimension_codes.include? d
+              # str << "  #{rdf_dimensions[j]} <code/#{d.downcase}/#{data[d][i]}> ;\n"
+              str << "  #{rdf_dimensions[j]} #{to_resource(data[d][i], options)} ;\n"
             else
-              str << "  prop:#{d} ns:#{to_resource(data[d][i], options)} ;\n"
+              str << "  #{rdf_dimensions[j]} #{to_literal(data[d][i], options)} ;\n"
             end
           }
 
-          measures.map{|m|
+          measures.each_with_index{|m,j|
             contains_nulls = contains_nulls | (data[m][i] == nil)
-            str << "  prop:#{m} #{to_literal(data[m][i], options)} ;\n" 
+            str << "  #{rdf_measures[j]} #{to_literal(data[m][i], options)} ;\n" 
             
           }
 
           str << "  .\n\n"
-          obs << str unless contains_nulls && !options[:encode_nulls]
-
+          if contains_nulls && !options[:encode_nulls]
+            if options[:raise_nils]
+              raise "missing component for observation, skipping: #{str}, "
+            elsif options[:whiny_nils]
+              puts "missing component for observation, skipping: #{str}, "
+            end
+          else
+            obs << str 
+          end
         }
         obs
       end
 
       def code_lists(codes, data, var, options={})
         options = defaults().merge(options)
+        rdf_measures, rdf_dimensions, rdf_codes  = generate_resources([], [], codes, options)
+        data = encode_data(codes, data, var, options)
         lists = []
-        codes.map{|code|
+        rdf_codes.map{|code|
+          if code[0] =~ /^<.+>$/
+            refcode = code[0][1..-2]
+          else
+            refcode = code[0]
+          end
           str = <<-EOF.unindent
-            code:#{code.downcase.capitalize} a rdfs:Class, owl:Class;
+            #{code[2]} a rdfs:Class, owl:Class;
               rdfs:subClassOf skos:Concept ;
-              rdfs:label "Code list for #{code} - codelist class"@en;
-              rdfs:comment "Specifies the #{code} for each observation";
-              rdfs:seeAlso code:#{code.downcase} .
-
-            code:#{code.downcase} a skos:ConceptScheme;
-              skos:prefLabel "Code list for #{code} - codelist scheme"@en;
-              rdfs:label "Code list for #{code} - codelist scheme"@en;
-              skos:notation "CL_#{code.upcase}";
-              skos:note "Specifies the #{code} for each observation";
+              rdfs:label "Code list for #{strip_prefixes(strip_uri(code[1]))} - codelist class"@en;
+              rdfs:comment "Specifies the #{strip_prefixes(strip_uri(code[1]))} for each observation";
+              rdfs:seeAlso #{code[1]} .
+
+            #{code[1]} a skos:ConceptScheme;
+              skos:prefLabel "Code list for #{strip_prefixes(strip_uri(code[1]))} - codelist scheme"@en;
+              rdfs:label "Code list for #{strip_prefixes(strip_uri(code[1]))} - codelist scheme"@en;
+              skos:notation "CL_#{strip_prefixes(strip_uri(code[1])).upcase}";
+              skos:note "Specifies the #{strip_prefixes(strip_uri(code[1]))} for each observation";
           EOF
-          data[code].uniq.map{|value|
+          data[refcode].uniq.map{|value|
             unless value == nil && !options[:encode_nulls]
-              str << "  skos:hasTopConcept <code/#{code.downcase}/#{to_resource(value,options)}> ;\n"
+              str << "  skos:hasTopConcept #{to_resource(value,options)} ;\n"
             end
           }
           
@@ -248,15 +334,22 @@ module R2RDF
 
       def concept_codes(codes, data, var, options={})
         options = defaults().merge(options)
+        rdf_measures, rdf_dimensions, rdf_codes  = generate_resources([], [], codes, options)
         concepts = []
-        codes.map{|code|
-          data[code].uniq.map{|value|
+        data = encode_data(codes, data, var, options)
+        rdf_codes.map{|code|
+          if code[0] =~ /^<.+>$/
+            refcode = code[0][1..-2]
+          else
+            refcode = code[0]
+          end
+          data[refcode].uniq.each_with_index{|value,i|
             unless value == nil && !options[:encode_nulls]
             concepts << <<-EOF.unindent
-              <code/#{code.downcase}/#{to_resource(value,options)}> a skos:Concept, code:#{code.downcase.capitalize};
-                skos:topConceptOf code:#{code.downcase} ;
-                skos:prefLabel "#{to_resource(value,options)}" ;
-                skos:inScheme code:#{code.downcase} .
+              #{to_resource(value,options)} a skos:Concept, #{code[2]};
+                skos:topConceptOf #{code[1]} ;
+                skos:prefLabel "#{strip_uri(data[refcode][i])}" ;
+                skos:inScheme #{code[1]} .
 
             EOF
             end
@@ -267,41 +360,9 @@ module R2RDF
       end
 
 
-      def to_resource(obj, options)
-        if obj.is_a? String
-          #TODO decide the right way to handle missing values, since RDF has no null
-          #probably throw an error here since a missing resource is a bigger problem
-          obj = "NA" if obj.empty?
-          
-          #TODO  remove special characters (faster) as well (eg '?')
-          obj.gsub(' ','_').gsub('?','')
-        elsif obj == nil && options[:encode_nulls]
-          '"NA"'
-        elsif obj.is_a? Numeric
-          #resources cannot be referred to purely by integer (?)
-          "n"+obj.to_s
-        else
-          obj
-        end
-      end
-
-      def to_literal(obj, options)
-        if obj.is_a? String
-          # Depressing that there's no more elegant way to check if a string is 
-          # a number...
-          if val = Integer(obj) rescue nil
-            val
-          elsif val = Float(obj) rescue nil
-            val
-          else
-            '"'+obj+'"'
-          end
-        elsif obj == nil && options[:encode_nulls]
-          #TODO decide the right way to handle missing values, since RDF has no null
-          '"NA"'
-        else
-          obj
-        end
+      def abbreviate_known(turtle_string)
+        #debug method
+        turtle_string.gsub(/<http:\/\/www\.rqtl\.org\/dc\/properties\/(\S+)>/, 'prop:\1').gsub(/<http:\/\/www.rqtl.org\/ns\/dc\/code\/(\S+)\/(\S+)>/, '<code/\1/\2>').gsub(/<http:\/\/www.rqtl.org\/dc\/dataset\/(\S+)\/code\/(\S+)>/, 'code:\2')
       end
     end
   end