publish_my_data 0.0.29 → 0.0.30

data/app/controllers/concerns/publish_my_data/data_download.rb

@@ -0,0 +1,22 @@
+module PublishMyData
+  module DataDownload
+    def find_latest_download_url_for_resource(resource)
+      # find the latest download for this dataset
+      # Note: filenames on s3 take the format: "<prefix>_<slug>_<time>.nt.zip"
+      # Only look for ones that were made on the same day as the the modified date, to restrict the results
+      # (v. small possibility of errors for changes aroung midnight, but unlikely people will be changing datasets then anyway!)
+      s3 = AWS::S3.new
+      downloads = s3.buckets[PublishMyData.downloads_s3_bucket].objects.with_prefix(resource.download_prefix).to_a
+
+      download_url = nil
+      if downloads.any?
+        download_url = downloads.last.public_url.to_s
+      end
+      download_url
+    end
+
+    def type_for_resource(resource)
+      resource.class.name.demodulize.underscore # ontology, concept_scheme or vocabulary
+    end
+  end
+end

data/app/controllers/concerns/publish_my_data/resource_rendering.rb

@@ -0,0 +1,40 @@
+module PublishMyData
+  module ResourceRendering
+    extend ActiveSupport::Concern
+
+    included do
+
+      private
+
+      def render_resource(resource)
+        respond_with(resource) do |format|
+          format.html do
+            resource.eager_load!
+
+            logger.debug(locals_for_resource(resource))
+            render template: template_for_resource(resource), locals: locals_for_resource(resource)
+          end
+        end
+      end
+
+      def template_for_resource(resource)
+        {
+          Dataset       => 'publish_my_data/datasets/show',
+          Ontology      => 'publish_my_data/ontologies/show',
+          ConceptScheme => 'publish_my_data/concept_schemes/show',
+          OntologyClass => 'publish_my_data/classes/show',
+          Property      => 'publish_my_data/properties/show',
+          Concept       => 'publish_my_data/concepts/show',
+          Resource      => 'publish_my_data/resources/show',
+          ThirdParty::Ontology =>       'publish_my_data/ontologies/show',
+          ThirdParty::ConceptScheme =>  'publish_my_data/concept_schemes/show'
+        }[resource.class]
+      end
+
+      def locals_for_resource(resource)
+        key = resource.class.name.demodulize.underscore.to_sym
+        {key => resource}
+      end
+    end
+  end
+end

data/app/controllers/concerns/publish_my_data/sparql_processing.rb

@@ -0,0 +1,81 @@
+module PublishMyData
+  module SparqlProcessing
+    extend ActiveSupport::Concern
+
+    included do
+
+      rescue_from PublishMyData::SparqlQueryExecutionException, :with => :show_sparql_execution_message
+
+      rescue_from PublishMyData::SparqlQueryMissingVariablesException, :with => :missing_variables
+
+      respond_to :html, :csv, :text, :nt, :ttl, :xml, :rdf, :json
+
+      private
+
+      def check_format
+        unless request.format
+          head :status => 406
+          return false
+        end
+      end
+
+      def build_sparql_query(query_text)
+        interpolations = request.params.reject{ |p| ['controller', 'action', 'page', 'per_page', 'id', 'commit' ,'utf8', 'query'].include?(p) }
+        PublishMyData::SparqlQuery.new(query_text, {
+          :request_format => request.format.to_sym,
+          :interpolations => interpolations
+        })
+      end
+
+      # process the sparql query, paginating if appropriate
+      def process_sparql_query(sparql_query)
+        if sparql_query.allow_pagination?
+
+          @pagination_params = SparqlPaginationParams.from_request(request)
+
+          # if there are pagination paramters, then use them
+          if @pagination_params.per_page && @pagination_params.page
+            sparql_query_result = sparql_query.paginate(@pagination_params.page, @pagination_params.per_page)
+            if is_request_html_format?
+              count = sparql_query.as_pagination_query(@pagination_params.page, @pagination_params.per_page, 1).count
+              @more_pages = (count > @pagination_params.per_page)
+            end
+          # otherwise just execute
+          else
+            sparql_query_result = @sparql_query.execute
+          end
+        else
+          # pagination not allowed - just execute.
+          sparql_query_result = @sparql_query.execute
+        end
+
+        add_json_p_callback(sparql_query_result)
+      end
+
+      def respond_with_error
+        respond_to do |format|
+          format.html { render 'publish_my_data/sparql/endpoint' }
+          format.any { render :text => @error_message, :status => 400 }
+        end
+      end
+
+      def missing_variables(e)
+        @error_message = e.message
+        respond_with_error
+      end
+
+      def show_sparql_execution_message(e)
+        @error_message = "There was a syntax error in your query: #{e.message}"
+        respond_with_error
+      end
+
+      def add_json_p_callback(result)
+        if request.format && request.format.json?
+          params[:callback] ? "#{params[:callback]}(#{result});" : result
+        else
+          result
+        end
+      end
+    end
+  end
+end

data/app/controllers/publish_my_data/datasets_controller.rb

@@ -2,36 +2,18 @@ require_dependency "publish_my_data/application_controller"
 
 module PublishMyData
   class DatasetsController < ApplicationController
+    include PublishMyData::DataDownload
 
     respond_to :html, :ttl, :rdf, :nt, :json, :text
 
     def dump
-      s3 = AWS::S3.new
       @dataset = Dataset.find_by_slug(params[:id])
-
-      # find the latest download for this dataset
-      # Note: filenames on s3 take the format: "dataset_data_<slug>_time.nt.zip"
-      # Only look for ones that were made on the same day as the the modified date, to restrict the results
-      # (v. small possibility of errors for changes aroung midnight, but unlikely people will be changing datasets then anyway!)
-      prefix = "dataset_data_#{@dataset.slug.gsub('/', '|')}_#{@dataset.modified.strftime("%Y%m%d")}"
-      downloads = s3.buckets[PublishMyData.dataset_downloads_s3_bucket].objects.with_prefix(prefix).to_a
-
-      # filter the downloads to only include ones with a timestamp equal to or after the dataset modified date.
-      # (ones older than this are out of date)
-      current_downloads = downloads.select do |d|
-        date_portion = d.public_url.to_s.split("_").last.split('.').first  #between last underscore and first dot.
-        file_timestamp = DateTime.parse(date_portion)
-        file_timestamp >= @dataset.modified
-      end
-
       # if we can't find a current download it's cos we haven't generated it yet since ds was modified
       # ... and we should 404.
-      if current_downloads.any?
-        latest_download = current_downloads.last
-        redirect_to latest_download.public_url.to_s
-      else
-        raise Tripod::Errors::ResourceNotFound
-      end
+      url = find_latest_download_url_for_resource(@dataset)
+      raise Tripod::Errors::ResourceNotFound unless url
+
+      redirect_to url
     end
 
     # /data?page=2&per_page=10

data/app/controllers/publish_my_data/information_resources_controller.rb

@@ -3,15 +3,37 @@ require_dependency "publish_my_data/application_controller"
 module PublishMyData
   class InformationResourcesController < ApplicationController
 
+    include ResourceRendering
+    include DataDownload
+
     respond_to :html, :ttl, :rdf, :nt, :json, :text
 
-    include PublishMyData::Concerns::Controllers::Resource
+    # http://example.com/data/blah
+    def data
+      uri = "http://#{PublishMyData.local_domain}/data/#{params[:id]}"
+      resource = PublishMyData::Resource.find(uri, local: true)
+      render_resource(resource)
+    end
 
-    def show
-      the_uri = "http://#{PublishMyData.local_domain}/data/#{params[:id]}"
-      render_resource_with_uri(the_uri)
+    # http://example.com/def/blah
+    def def
+      uri = "http://#{PublishMyData.local_domain}/def/#{params[:id]}"
+      resource = PublishMyData::Resource.find(uri, local: true)
+      render_resource(resource)
     end
 
+    # http://example.com/def/blah/dump
+    def dump
+      uri = "http://#{PublishMyData.local_domain}/def/#{params[:id]}"
+      resource = PublishMyData::Resource.find(uri, local: true)
+
+      # if we can't find a current download it's cos we haven't generated it yet since ds was modified
+      # ... and we should 404.
+      url = find_latest_download_url_for_resource(resource)
+      raise Tripod::Errors::ResourceNotFound unless url
+
+      redirect_to url
+    end
   end
 
 end

data/app/controllers/publish_my_data/queries_controller.rb

@@ -3,7 +3,7 @@ require_dependency "publish_my_data/application_controller"
 module PublishMyData
   class QueriesController < ApplicationController
 
-    include PublishMyData::Concerns::Controllers::Sparql
+    include PublishMyData::SparqlProcessing
 
     before_filter :check_format, :only => [:show]
 

data/app/controllers/publish_my_data/resources_controller.rb

@@ -3,7 +3,7 @@ require_dependency "publish_my_data/application_controller"
 module PublishMyData
   class ResourcesController < ApplicationController
 
-    include PublishMyData::Concerns::Controllers::Resource
+    include ResourceRendering
 
     respond_to :html, :ttl, :rdf, :nt, :json, :text
 
@@ -30,7 +30,8 @@ module PublishMyData
 
       if uri.present?
         begin
-          render_resource_with_uri(uri)
+          resource = PublishMyData::Resource.find(uri, local: uri.starts_with?('http://' + PublishMyData.local_domain))
+          render_resource(resource)
         rescue Tripod::Errors::ResourceNotFound
           # if it's not there
           respond_to do |format|
@@ -58,13 +59,8 @@ module PublishMyData
     # http://example.com/doc/blah
     def doc
       uri = Resource.uri_from_host_and_doc_path(request.host, params[:path], params[:format])
-      render_resource_with_uri(uri)
-    end
-
-    # http://example.com/def/blah
-    def definition
-      uri = 'http://' + request.host + '/def/' + params[:path]
-      render_resource_with_uri(uri)
+      resource = PublishMyData::Resource.find(uri)
+      render_resource(resource)
     end
 
     private

data/app/controllers/publish_my_data/sparql_controller.rb

@@ -3,7 +3,7 @@ require_dependency "publish_my_data/application_controller"
 module PublishMyData
   class SparqlController < ApplicationController
 
-    include PublishMyData::Concerns::Controllers::Sparql
+    include PublishMyData::SparqlProcessing
 
     before_filter :check_format, :only => [:endpoint]
 

data/app/controllers/publish_my_data/vocabularies_controller.rb

@@ -0,0 +1,18 @@
+require_dependency "publish_my_data/application_controller"
+
+module PublishMyData
+  class VocabulariesController < ApplicationController
+    include PublishMyData::DataDownload
+
+    def dump
+      vocabulary = Vocabulary.find_by_data_dump(request.url)
+
+      # if we can't find a current download it's cos we haven't generated it yet since ds was modified
+      # ... and we should 404.
+      url = find_latest_download_url_for_resource(vocabulary)
+      raise Tripod::Errors::ResourceNotFound unless url
+
+      redirect_to url
+    end
+  end
+end

data/app/models/concerns/publish_my_data/all_features.rb

@@ -0,0 +1,79 @@
+module PublishMyData
+  module AllFeatures
+    extend ActiveSupport::Concern
+    
+    included do
+      include PublishMyData::BasicFeatures
+      
+      # basics
+      field :title, RDF::DC.title
+      field :description, RDF::DC.description # long desc
+
+      # licence, owner, contact
+      field :publisher, RDF::DC.publisher, :is_uri => true # value is a URI of a publisher
+      field :license, RDF::DC.license, :is_uri => true # value is URI of where licence is defined.
+      field :contact_email, RDF::PMD_DS.contactEmail, :is_uri => true
+      # NOTE: for contact, use :publisher's foaf:mbox value (in this metadata graph).
+
+      # quality, updates, maintenance
+      field :issued, RDF::DC.issued, :datatype => RDF::XSD.dateTime # value is DateTime literal
+      field :modified, RDF::DC.modified, :datatype => RDF::XSD.dateTime # value is DateTime literal
+   #   field :update_periodicity, RDF::DC.accrualPeriodicity # waiting for response on what the value should be.
+
+      # where to get it
+      field :data_dump, RDF::VOID.dataDump, :is_uri => true # full download URI
+
+       # what the data is about
+      field :tags, RDF::DCAT.keyword, :multivalued => true # values are string literals
+
+      # field :spatial_coverage, RDF::DC.spatial # value is a URI for region covered, e.g. England.
+      # field :temporal_coverage, RDF::DC.temporal # value is a time interval URI
+      # field :spatial_granularity, RDF::DC.spatial # value is class of the objects of refArea
+      # field :temporal_granularity, RDF::DC.temporal # value is class of objects of refPeriod
+
+      field :size, RDF::VOID.triples # value is integer.
+      field :replaced_by, RDF::DC.isReplacedBy, is_uri: true
+
+      cattr_accessor :_DEPRECATED_RDF_TYPE
+    end
+
+    def deprecated?
+      self.rdf_type.include?(self.class.get_deprecated_rdf_type)
+    end
+
+    def slug
+      self.class.slug_from_uri(self.uri)
+    end
+
+    def resource_type
+      self.class.name.demodulize.underscore
+    end
+
+    def download_prefix
+      "#{self.resource_type}_data_#{self.slug.gsub('/', '|')}_#{self.modified.strftime("%Y%m%d")}"
+    end
+
+    module ClassMethods
+      def deprecated_rdf_type(type)
+        self._DEPRECATED_RDF_TYPE = type
+      end
+
+      def get_deprecated_rdf_type
+        self._DEPRECATED_RDF_TYPE
+      end
+
+      def uri_from_slug(slug)
+        # Implement!
+      end
+
+      def slug_from_uri(uri)
+        root_uri = self.uri_from_slug('')
+        uri.to_s.gsub(root_uri, '')
+      end
+
+      def find_by_slug(slug)
+        find(uri_from_slug(slug))
+      end
+    end
+  end
+end

data/app/models/concerns/publish_my_data/basic_features.rb

@@ -0,0 +1,21 @@
+module PublishMyData
+  module BasicFeatures
+    extend ActiveSupport::Concern
+
+    included do
+      field :label, RDF::RDFS.label # same as the title
+      field :comment, RDF::RDFS.comment #short desc
+      field :data_graph_uri, RDF::PMD_DS.graph, is_uri: true
+    end
+
+    # Is this resource in the host domain?
+    def in_domain?(domain)
+      uri.starts_with?("http://" + domain)
+    end
+
+    def eager_load!
+      eager_load_object_triples!(labels_only: true)
+      eager_load_predicate_triples!(labels_only: true)
+    end
+  end
+end

data/app/models/concerns/publish_my_data/dataset_powers.rb

@@ -0,0 +1,130 @@
+module PublishMyData
+  module DatasetPowers
+    extend ActiveSupport::Concern
+
+    included do
+      include PublishMyData::AllFeatures
+      field :theme, RDF::DCAT.theme, :is_uri => true
+    end
+
+    def metadata_graph_uri
+      self.class.metadata_graph_uri(self.slug)
+    end
+
+    def to_param
+      slug
+    end
+
+    def resources_in_dataset_criteria
+      Resource.all.graph(self.data_graph_uri)
+    end
+
+    def types
+      @types ||= RdfType.where('?s a ?uri').graph(self.data_graph_uri).resources
+    end
+
+    def type_count(type_uri)
+      count_query = "SELECT ?uri WHERE { GRAPH <#{self.data_graph_uri.to_s}> { ?uri a <#{type_uri.to_s}> } }"
+      SparqlQuery.new(count_query).count  
+    end
+
+    def resource_count
+      self.types.map{|t| type_count(t.uri)}.sum
+    end
+
+    def example_resources
+      return @example_resources if @example_resources
+
+      resource_queries = self.types.map do |t|
+        "{ SELECT DISTINCT ?uri WHERE { ?uri a <#{t.uri.to_s}> } LIMIT 1 }"
+      end
+      query =  "SELECT ?uri WHERE { GRAPH <#{self.data_graph_uri.to_s}> {"
+      query << resource_queries.join(" UNION ")
+      query << "}}"
+      @example_resources = ExampleResource.find_by_sparql(query)
+      @example_resources.each {|r| r.eager_load!}
+      @example_resources
+    end
+
+    def ontologies
+      return @ontologies if @ontologies
+
+      query =  "SELECT DISTINCT ?uri WHERE {"
+      query << "  GRAPH <#{self.data_graph_uri.to_s}> {?s ?p ?o}"
+      query << "  { ?p <http://www.w3.org/2000/01/rdf-schema#isDefinedBy> ?uri } UNION { ?o <http://www.w3.org/2000/01/rdf-schema#isDefinedBy> ?uri }"
+      query << "  ?uri a <http://www.w3.org/2002/07/owl#Ontology>."
+      query << "}"
+      @ontologies = Ontology.find_by_sparql(query)
+    end
+
+    def concept_schemes
+      return @concept_schemes if @concept_schemes
+      
+      query =  "SELECT DISTINCT ?uri WHERE {"
+      query << "  GRAPH <#{self.data_graph_uri.to_s}> {?s ?p ?o}"
+      query << "  { ?p <http://www.w3.org/2004/02/skos/core#inScheme> ?uri } UNION { ?o <http://www.w3.org/2004/02/skos/core#inScheme> ?uri }"
+      query << "  ?uri a <http://www.w3.org/2004/02/skos/core#ConceptScheme>"
+      query << "}"
+      @concept_schemes = ConceptScheme.find_by_sparql(query)
+    end
+
+    def theme_obj
+      Theme.find(self.theme.to_s) rescue nil
+    end
+
+    module ClassMethods
+      include PublishMyData::AllFeatures::ClassMethods
+
+      # this is the graph that dataset metadata goes in.
+      def metadata_graph_uri(slug)
+        "#{data_graph_uri(slug)}/metadata"
+      end
+
+      # this is the dataset that the actual data will go in
+      def data_graph_uri(slug)
+        "http://#{PublishMyData.local_domain}/graph/#{slug}"
+      end
+
+      def uri_from_data_graph_uri(data_graph_uri)
+        data_graph_uri.to_s.sub("/graph/", "/data/")
+      end
+
+      def uri_from_slug(slug)
+        "http://#{PublishMyData.local_domain}/data/#{slug}"
+      end
+
+      # Criteria etc.
+
+      def ordered_by_title
+        all.where("?uri <#{RDF::DC.title}> ?title").order("?title")
+      end
+
+      def deprecation_last_query_str
+        "
+        SELECT ?uri where {
+          # this bit is all the non-deprecated ones
+          {
+            SELECT * WHERE {
+              ?uri a <http://publishmydata.com/def/dataset#Dataset> .
+              ?uri <#{RDF::DC.title}> ?title . # select title so we can order
+              MINUS {
+                ?uri a <http://publishmydata.com/def/dataset#DeprecatedDataset>
+              }
+            }
+            ORDER BY ?title
+          }
+          UNION
+          # this bit is all the deprecated ones
+          {
+            SELECT * WHERE {
+              ?uri a <http://publishmydata.com/def/dataset#DeprecatedDataset> .
+              ?uri <#{RDF::DC.title}> ?title . # select title so we can order
+            }
+            ORDER BY ?title
+          }
+        }
+        "
+      end
+    end
+  end
+end