bio-publisci 0.0.8 → 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e7e64a90c4e9a15927e2ef67477298d50d22358e
+  data.tar.gz: 18c0ccecba26dc09e9604966b992f2444dbf4b0c
+SHA512:
+  metadata.gz: 7bdfa9be8aee03c937b8a4298f05589035ebfbe2ffe8d0ece67da239580f0ab11b192dc3d2e48cb7a2efde2e78ee31f2cc20ba55afdf68c60565a31c8c3251ee
+  data.tar.gz: 1d0a10523a45cf231f521f4026fe7672e29143ca68a0d56af8bcc4df4a4b433b286c81eae7d4008944b93e4f9806732aec95b43a9faf2b885673686722eba7a9

data/Gemfile

@@ -21,6 +21,16 @@ end
 gem 'rdf'
 gem 'rdf-4store', :git => "https://github.com/fumi/rdf-4store.git"
 gem 'rdf-turtle'
+gem 'rdf-rdfxml'
+gem 'json-ld'
 gem 'rserve-client'
 gem 'sparql'
 
+gem 'slop'
+
+gem 'sinatra'
+gem 'sinatra-contrib'
+gem 'sinatra-linkeddata'
+gem 'sinatra-cross_origin'
+gem 'rack-flash3'
+gem 'rest-client'

data/Rakefile

@@ -21,7 +21,7 @@ Jeweler::Tasks.new do |gem|
   gem.description = %Q{A toolkit for publishing scientific results and datasets using RDF, OWL, and related technologies }
   gem.email = "wstrinz@gmail.com"
   gem.authors = ["Will Strinz"]
-  gem.version = "0.0.8"
+  gem.version = "0.1.0"
 
   # dependencies defined in Gemfile
 end

data/bin/bio-publisci-server

@@ -0,0 +1,50 @@
+#!/usr/bin/env ruby
+require 'cgi'
+require 'yaml'
+
+require 'bio-publisci'
+require 'slop'
+
+require 'sinatra/base'
+require "sinatra/linkeddata"
+require "sinatra/cross_origin"
+require 'rack-flash'
+
+class PubliSciServer < Sinatra::Base
+  configure do
+    enable :sessions
+    enable :cross_origin
+  end
+
+  set :views, File.dirname(__FILE__) + '/../server/views'
+  # set :repository, RDF::Repository.new
+  set :sudo_pass, nil
+
+  use Rack::Flash
+
+  require_relative '../server/helpers.rb'
+
+  if ARGV[0]
+    opts = Slop.parse do
+      banner 'Usage: bio-publisci-server [options]'
+
+      on 'uri', 'Store URI', argument: :optional
+      on 't', 'type', 'The repository type', argument: :optional
+      on 'a', 'sudo_pass', 'Password for sudo (currently needed to clear 4store)', argument: :optional
+      on 'p', 'port', 'Sinatra port', argument: :optional
+      on 'o', 'bind', 'Bind address ', argument: :optional
+    end
+
+    configure_server(opts)
+  else
+    # set :repository, RDF::FourStore::Repository.new('http://localhost:8080')
+    set :repository, RDF::Repository.new
+
+    settings.repository.load(File.dirname(__FILE__) + '/../resources/primer.ttl')
+  end
+
+  require_relative '../server/routes.rb'
+
+
+  run!
+end

data/features/reader_steps.rb

@@ -1,7 +1,7 @@
 require_relative '../lib/bio-publisci.rb'
 
 Given /^a (.*) generator$/ do |generator|
-	@generator = PubliSci::Reader.const_get(generator).new
+	@generator = PubliSci::Readers.const_get(generator).new
 end
 
 When /^I ask for its methods$/ do

data/lib/bio-publisci.rb

@@ -1,12 +1,19 @@
 # This is temporary, just to help w/ development so I don't have to rewrite r2rdf.rb to be
 # a standard gem base yet. Also load s the files instead of require for easy reloading
 require 'tempfile'
-require 'rdf'
+require 'fileutils'
 require 'csv'
-require 'rserve'
+
+require 'rdf'
 require 'sparql'
 require 'sparql/client'
 require 'rdf/turtle'
+require 'rdf/rdfxml'
+require 'json/ld'
+
+require 'rserve'
+require 'rest-client'
+
 begin
 	require 'spira'
 rescue LoadError
@@ -27,6 +34,7 @@ load_folder('bio-publisci/mixins')
 load File.dirname(__FILE__) + '/bio-publisci/dataset/interactive.rb'
 load File.dirname(__FILE__) + '/bio-publisci/query/query_helper.rb'
 load File.dirname(__FILE__) + '/bio-publisci/parser.rb'
+load File.dirname(__FILE__) + '/bio-publisci/post_processor.rb'
 load File.dirname(__FILE__) + '/bio-publisci/r_client.rb'
 load File.dirname(__FILE__) + '/bio-publisci/analyzer.rb'
 load File.dirname(__FILE__) + '/bio-publisci/store.rb'
@@ -39,6 +47,7 @@ load File.dirname(__FILE__) + '/bio-publisci/output.rb'
 load File.dirname(__FILE__) + '/bio-publisci/metadata/prov/element.rb'
 load File.dirname(__FILE__) + '/bio-publisci/metadata/prov/prov.rb'
 load File.dirname(__FILE__) + '/bio-publisci/writers/base.rb'
+load File.dirname(__FILE__) + '/bio-publisci/readers/base.rb'
 
 
 load_folder('bio-publisci/dsl')

data/lib/bio-publisci/datacube_model.rb

@@ -1,107 +1,111 @@
 require 'rdf/4store'
+begin
+  require 'spira'
+  module PubliSci
+    module ORM
+      # class Observation < Spira::Base
+      #   type RDF::URI.new('http://purl.org/linked-data/cube#Observation')
+      #   property :label, predicate: RDFS.label
+
+      # end
+      QB ||= RDF::Vocabulary.new(RDF::URI.new('http://purl.org/linked-data/cube#'))
+
+      class Component < Spira::Base
+        type QB.ComponentSpecification
+        property :dimension, predicate: QB.dimension # RDF::URI.new('http://purl.org/linked-data/cube#dimension')
+        property :measure, predicate: QB.measure # RDF::URI.new('http://purl.org/linked-data/cube#measure')
+      end
 
-module PubliSci
-  module ORM
-    # class Observation < Spira::Base
-    #   type RDF::URI.new('http://purl.org/linked-data/cube#Observation')
-    #   property :label, predicate: RDFS.label
-
-    # end
-    QB ||= RDF::Vocabulary.new(RDF::URI.new('http://purl.org/linked-data/cube#'))
-
-    class Component < Spira::Base
-      type QB.ComponentSpecification
-      property :dimension, predicate: QB.dimension # RDF::URI.new('http://purl.org/linked-data/cube#dimension')
-      property :measure, predicate: QB.measure # RDF::URI.new('http://purl.org/linked-data/cube#measure')
-    end
+      class DataStructureDefinition < Spira::Base
+        type QB.DataStructureDefinition
+        property :label, predicate: RDFS.label
+        has_many :component, predicate: QB.component
+      end
 
-    class DataStructureDefinition < Spira::Base
-      type QB.DataStructureDefinition
-      property :label, predicate: RDFS.label
-      has_many :component, predicate: QB.component
-    end
+      class DataSet < Spira::Base
+        type QB.DataSet
+        property :label, predicate: RDFS.label
+        property :structure, predicate: QB.structure
+      end
 
-    class DataSet < Spira::Base
-      type QB.DataSet
-      property :label, predicate: RDFS.label
-      property :structure, predicate: QB.structure
-    end
+      class Dimension < Spira::Base
+        type QB.DimensionProperty
+        property :range, predicate: RDFS.range
+        property :label, predicate: RDFS.label
+      end
 
-    class Dimension < Spira::Base
-      type QB.DimensionProperty
-      property :range, predicate: RDFS.range
-      property :label, predicate: RDFS.label
-    end
+      class Measure < Spira::Base
+        type QB.MeasureProperty
+        property :label, predicate: RDFS.label
+      end
 
-    class Measure < Spira::Base
-      type QB.MeasureProperty
-      property :label, predicate: RDFS.label
-    end
 
+      def load_repo(repo)
+        raise "Not an RDF::Repository - #{repo}" unless repo.is_a? RDF::Repository
+        Spira.add_repository :default, repo
+      end
 
-    def load_repo(repo)
-      raise "Not an RDF::Repository - #{repo}" unless repo.is_a? RDF::Repository
-      Spira.add_repository :default, repo
-    end
+      # def observation
+      #   unless PubliSci::ORM.const_defined?("Observation")
+      #     obs = Class.new(Spira::Base) do
+      #       type RDF::URI.new('http://purl.org/linked-data/cube#Observation')
+
+      #       property :structure, predicate: QB.dataSet
+
+      #       ((Dimension.each.to_a | Measure.each.to_a) || []).each{|component|
+      #         property strip_uri(component.subject.to_s), predicate: component.subject
+      #       }
+      #     end
+      #     PubliSci::ORM.const_set("Observation",obs)
+      #   end
+      #   Observation
+      # end
+
+      class Observation < Spira::Base
+        type QB.Observation
+        property :label, predicate: RDFS.label
+        property :dataset, predicate: QB.dataSet
+
+        def load_properties
+          comps = dataset.as(DataSet).structure.as(DataStructureDefinition).component.map{|comp| comp.as(Component)}
+          props = comps.map{|comp| comp.dimension ? comp.dimension.as(Dimension) : comp.measure.as(Measure) }
+          props.each{|prop|
+            ss =  strip_uri(prop.subject.to_s)
+
+            self.class.property ss.to_sym, predicate: prop.subject
+          }
+        end
+
+        # for testing; DRY up eventually
+        def strip_uri(uri)
+          uri = uri.to_s.dup
+          uri[-1] = '' if uri[-1] == '>'
+          uri.to_s.split('/').last.split('#').last
+        end
+
+        # def method_missing(meth, *args, &block)
+        #         if meth.to_s =~ /^find_by_(.+)$/
+        #           run_find_by_method($1, *args, &block)
+        #         else
+        #           super # You *must* call super if you don't handle the
+        #                 # method, otherwise you'll mess up Ruby's method
+        #                 # lookup.
+        #         end
+        #       end
+      end
 
-    # def observation
-    #   unless PubliSci::ORM.const_defined?("Observation")
-    #     obs = Class.new(Spira::Base) do
-    #       type RDF::URI.new('http://purl.org/linked-data/cube#Observation')
-
-    #       property :structure, predicate: QB.dataSet
-
-    #       ((Dimension.each.to_a | Measure.each.to_a) || []).each{|component|
-    #         property strip_uri(component.subject.to_s), predicate: component.subject
-    #       }
-    #     end
-    #     PubliSci::ORM.const_set("Observation",obs)
-    #   end
-    #   Observation
-    # end
-
-    class Observation < Spira::Base
-      type QB.Observation
-      property :label, predicate: RDFS.label
-      property :dataset, predicate: QB.dataSet
-
-      def load_properties
-        comps = dataset.as(DataSet).structure.as(DataStructureDefinition).component.map{|comp| comp.as(Component)}
-        props = comps.map{|comp| comp.dimension ? comp.dimension.as(Dimension) : comp.measure.as(Measure) }
-        props.each{|prop|
-          ss =  strip_uri(prop.subject.to_s)
-
-          self.class.property ss.to_sym, predicate: prop.subject
-        }
+      def reload_observation
+        PubliSci::ORM.send(:remove_const, "Observation")
+        observation
       end
 
-      # for testing; DRY up eventually
       def strip_uri(uri)
         uri = uri.to_s.dup
         uri[-1] = '' if uri[-1] == '>'
         uri.to_s.split('/').last.split('#').last
       end
-
-      # def method_missing(meth, *args, &block)
-      #         if meth.to_s =~ /^find_by_(.+)$/
-      #           run_find_by_method($1, *args, &block)
-      #         else
-      #           super # You *must* call super if you don't handle the
-      #                 # method, otherwise you'll mess up Ruby's method
-      #                 # lookup.
-      #         end
-      #       end
-    end
-
-    def reload_observation
-      PubliSci::ORM.send(:remove_const, "Observation")
-      observation
-    end
-
-    def strip_uri(uri)
-      uri = uri.to_s.dup
-      uri[-1] = '' if uri[-1] == '>'
-      uri.to_s.split('/').last.split('#').last
     end
   end
+rescue LoadError
+  # puts "Skipping ORM load"
 end

data/lib/bio-publisci/dataset/ORM/data_cube_orm.rb

@@ -4,7 +4,6 @@ module PubliSci
       class DataCube
         extend PubliSci::Dataset::DataCube
         extend PubliSci::Analyzer
-        # extend PubliSci::Metadata
         extend PubliSci::Query
         extend PubliSci::Parser
 

data/lib/bio-publisci/dataset/data_cube.rb

@@ -13,7 +13,7 @@ module PubliSci
       {
         type: :dataframe,
         encode_nulls: false,
-        base_url: "http://www.rqtl.org",
+        base_url: "http://onto.strinz.me",
       }
       end
 
@@ -32,12 +32,10 @@ module PubliSci
 
         newd = dimensions.map{|d|
             if d =~ /^http:\/\//
-              # newc << "<#{d}>" if codes.include? d
               "<#{d}>"
             elsif d =~ /^[a-zA-z]+:[a-zA-z]+$/
               d
             else
-              # newc << "prop:#{d}" if codes.include? d
               "prop:#{d}"
             end
         }
@@ -60,9 +58,10 @@ module PubliSci
         [newm, newd, newc]
       end
 
-      def component_gen(args,options={})
+      def component_gen(args,var,options={})
         args = Array[args].flatten
-        args.map{|arg| arg.gsub("prop:","cs:").gsub(%r{<#{options[:base_url]}/.+/(\w.+)>$},'cs:'+'\1')}
+        args = args.map{|arg| arg.gsub("prop:","cs:").gsub(%r{<#{options[:base_url]}/.+/(\w.+)>$},'cs:'+'\1')}
+        args.map{|arg| arg.gsub(%r{<http://(.+)>},"<#{options[:base_url]}/dc/dataset/#{var}/cs/"+'\1'+'>')}
       end
 
       def encode_data(codes,data,var,options={})
@@ -76,7 +75,7 @@ module PubliSci
               elsif val =~ /^[a-zA-z]+:[a-zA-z]+$/
                 val
               else
-                "<code/#{k.downcase}/#{val}>"
+                "<code/#{k.downcase}/#{sanitize(val).first}>"
               end
             }
           else
@@ -120,12 +119,12 @@ module PubliSci
         options = defaults().merge(options)
         base = options[:base_url]
         <<-EOF.unindent
-        @base <#{base}/ns/dc/> .
-        @prefix ns:    <#{base}/ns/dataset/#{var}/> .
+        @base <#{base}/dc/dataset/#{var}/> .
+        @prefix ns:    <#{base}/dc/dataset/#{var}/> .
         @prefix qb:    <http://purl.org/linked-data/cube#> .
         @prefix rdf:   <http://www.w3.org/1999/02/22-rdf-syntax-ns#> .
         @prefix rdfs:  <http://www.w3.org/2000/01/rdf-schema#> .
-        @prefix prop:  <#{base}/dc/properties/> .
+        @prefix prop:  <#{base}/properties/> .
         @prefix dct:   <http://purl.org/dc/terms/> .
         @prefix xsd:   <http://www.w3.org/2001/XMLSchema#> .
         @prefix cs:    <#{base}/dc/dataset/#{var}/cs/> .
@@ -143,8 +142,8 @@ module PubliSci
         var = sanitize([var]).first
         options = defaults().merge(options)
         rdf_measures, rdf_dimensions, rdf_codes  = generate_resources(measures, dimensions, codes, options)
-        cs_dims = component_gen(rdf_dimensions,options)  #rdf_dimensions.map{|d| d.gsub('prop:','cs:')}
-        cs_meas = component_gen(rdf_measures,options) #rdf_measures.map!{|m| m.gsub('prop:','cs:')}
+        cs_dims = component_gen(rdf_dimensions,var,options)  #rdf_dimensions.map{|d| d.gsub('prop:','cs:')}
+        cs_meas = component_gen(rdf_measures,var,options) #rdf_measures.map!{|m| m.gsub('prop:','cs:')}
         str = "ns:dsd-#{var} a qb:DataStructureDefinition;\n"
         cs_dims.map{|d|
           str << "  qb:component #{d} ;\n"
@@ -172,16 +171,14 @@ module PubliSci
       def component_specifications(measure_names, dimension_names, codes, var, options={})
         options = defaults().merge(options)
         rdf_measures, rdf_dimensions, rdf_codes  = generate_resources(measure_names, dimension_names, codes, options)
-        cs_dims = component_gen(rdf_dimensions,options)
-        cs_meas = component_gen(rdf_measures,options)
-        # cs_dims = rdf_dimensions.map{|d| d.gsub('prop:','cs:')}
-        # cs_meas = rdf_measures.map{|m| m.gsub('prop:','cs:')}
+        cs_dims = component_gen(rdf_dimensions,var,options)
+        cs_meas = component_gen(rdf_measures,var,options)
         specs = []
 
           rdf_dimensions.each_with_index.map{|d,i|
           specs << <<-EOF.unindent
             #{cs_dims[i]} a qb:ComponentSpecification ;
-              rdfs:label "#{strip_prefixes(strip_uri(dimension_names[i]))} Component" ;
+              rdfs:label "#{strip_prefixes(strip_uri(dimension_names[i]))}" ;
               qb:dimension #{d} .
 
             EOF
@@ -190,7 +187,7 @@ module PubliSci
           rdf_measures.each_with_index.map{|n,i|
             specs << <<-EOF.unindent
               #{cs_meas[i]} a qb:ComponentSpecification ;
-                rdfs:label "#{strip_prefixes(strip_uri(measure_names[i]))} Component" ;
+                rdfs:label "#{strip_prefixes(strip_uri(measure_names[i]))}" ;
                 qb:measure #{n} .
 
               EOF
@@ -214,6 +211,7 @@ module PubliSci
 
         rdf_dimensions.each_with_index{|d,i|
           if dimension_codes.include?(dimensions[i])
+
             code = rdf_codes[dimension_codes.index(dimensions[i])]
             props << <<-EOF.unindent
             #{d} a rdf:Property, qb:DimensionProperty ;
@@ -225,9 +223,13 @@ module PubliSci
           else
             props << <<-EOF.unindent
             #{d} a rdf:Property, qb:DimensionProperty ;
-              rdfs:label "#{strip_prefixes(strip_uri(d))}"@en .
-
+              rdfs:label "#{strip_prefixes(strip_uri(d))}"@en ;
             EOF
+            if options[:ranges] && options[:ranges][dimension[i]]
+              props.last << "\n  rdfs:range #{options[:ranges][dimensions[i]]} .\n\n"
+            else
+              props.last[-2] = ".\n"
+            end
           end
           }
 
@@ -239,14 +241,19 @@ module PubliSci
         rdf_measures = generate_resources(measures, [], [], options)[0]
         props = []
 
-          rdf_measures.map{ |m|
+        rdf_measures.each_with_index{ |m,i|
 
-            props <<  <<-EOF.unindent
-            #{m} a rdf:Property, qb:MeasureProperty ;
-              rdfs:label "#{strip_prefixes(strip_uri(m))}"@en .
+          props <<  <<-EOF.unindent
+          #{m} a rdf:Property, qb:MeasureProperty ;
+            rdfs:label "#{strip_prefixes(strip_uri(m))}"@en ;
+          EOF
 
-            EOF
-            }
+          if options[:ranges] && options[:ranges][measures[i]]
+            props.last << "  rdfs:range #{options[:ranges][measures[i]]} .\n\n"
+          else
+            props.last[-2] = ".\n"
+          end
+        }
 
         props
       end
@@ -255,7 +262,11 @@ module PubliSci
         var = sanitize([var]).first
         measures = sanitize(measures)
         dimensions = sanitize(dimensions)
+
+        data.each{|k,v| data[k]=Array(v)}
+        observation_labels = Array(observation_labels)
         options = defaults().merge(options)
+
         rdf_measures, rdf_dimensions, rdf_codes  = generate_resources(measures, dimensions, codes, options)
         data = encode_data(codes, data, var, options)
         obs = []
@@ -269,7 +280,7 @@ module PubliSci
         }
 
         observation_labels.each_with_index.map{|r, i|
-          contains_nulls = false
+          # contains_nulls = false
           str = <<-EOF.unindent
           ns:obs#{r} a qb:Observation ;
             qb:dataSet ns:dataset-#{var} ;
@@ -277,33 +288,57 @@ module PubliSci
 
           str << "  rdfs:label \"#{r}\" ;\n" unless options[:no_labels]
 
+          obs_index = 0
+          obs_nodes = []
+
           dimensions.each_with_index{|d,j|
-            contains_nulls = contains_nulls | (data[d][i] == nil)
+            contains_nulls = (data[d][i] == nil)
 
-            if dimension_codes.include? d
-              # str << "  #{rdf_dimensions[j]} <code/#{d.downcase}/#{data[d][i]}> ;\n"
-              str << "  #{rdf_dimensions[j]} #{to_resource(data[d][i], options)} ;\n"
-            else
-              str << "  #{rdf_dimensions[j]} #{to_literal(data[d][i], options)} ;\n"
+            unless contains_nulls && !options[:encode_nulls]
+              if is_complex?(data[d][i])
+                str << "  #{rdf_dimensions[j]} #{add_node(obs_index,add_node(r))} ;\n"
+                obs_nodes << encode_value(data[d][i], options, obs_index, add_node(r))
+              else
+                str << "  #{rdf_dimensions[j]} #{encode_value(data[d][i], options)} ;\n"
+              end
             end
+
+            obs_index += 1
           }
 
           measures.each_with_index{|m,j|
-            contains_nulls = contains_nulls | (data[m][i] == nil)
-            str << "  #{rdf_measures[j]} #{to_literal(data[m][i], options)} ;\n"
+            contains_nulls = (data[m][i] == nil)
+
+            unless contains_nulls && !options[:encode_nulls]
+              if is_complex?(data[m][i])
+                str << "  #{rdf_measures[j]} #{add_node(obs_index,add_node(r))} ;\n"
+                val = encode_value(data[m][i], options, obs_index, add_node(r))
+
+                if val.last.is_a? Array
+                  unless val.last.last[-2] == "."
+                    val.last.last << ".\n"
+                  end
+                end
+                
+                obs_nodes << val 
+              else
+                str << "  #{rdf_measures[j]} #{encode_value(data[m][i], options)} ;\n"
+              end
+            end
 
+            obs_index += 1
           }
 
           str << "  .\n\n"
-          if contains_nulls && !options[:encode_nulls]
-            if options[:raise_nils]
-              raise "missing component for observation, skipping: #{str}, "
-            elsif options[:whiny_nils]
-              puts "missing component for observation, skipping: #{str}, "
-            end
-          else
-            obs << str
+
+          if obs_nodes.size > 0
+            flatted = obs_nodes.flatten
+            str << turtle_indent(flatted.join("\n"))
+            str << "  \n\n"
           end
+
+          obs << str
+
         }
         obs
       end
@@ -356,7 +391,6 @@ module PubliSci
           else
             refcode = code[0]
           end
-          # puts data[refcode].uniq
           data[refcode].uniq.each_with_index{|value,i|
             unless value == nil && !options[:encode_nulls]
             concepts << <<-EOF.unindent
@@ -377,7 +411,7 @@ module PubliSci
       def abbreviate_known(turtle_string)
         #debug method
         # puts turtle_string
-        turtle_string.gsub(/<http:\/\/www\.rqtl\.org\/dc\/properties\/(\S+)>/, 'prop:\1').gsub(/<http:\/\/www.rqtl.org\/ns\/dc\/code\/(\S+)\/(\S+)>/, '<code/\1/\2>').gsub(/<http:\/\/www.rqtl.org\/dc\/dataset\/(\S+)\/code\/(\S+)>/, 'code:\2')
+        turtle_string.gsub(/<http:\/\/www\.rqtl\.org\/dc\/properties\/(\S+)>/, 'prop:\1').gsub(/<http:\/\/www.rqtl.org\/ns\/dc\/code\/(\S+)\/(\S+)>/, '<code/\1/\2>').gsub(/<http:\/\/www.rqtl.org\/dc\/dataset\/(\S+)\/code\/(\w+)>/, 'code:\2').gsub(/<http:\/\/www.rqtl.org\/dc\/dataset\/(\S+)\/code\/(\S+)>/, '<code/' + '\2' +'>')
       end
     end
   end