publish_my_data 0.0.32 → 1.0.0

data/app/models/publish_my_data/data_cube/cube.rb

@@ -0,0 +1,358 @@
+module PublishMyData
+  module DataCube
+
+    class Cube
+
+      include PublishMyData::CubeResults
+
+      attr_reader :dataset
+
+      def initialize(dataset)
+        @dataset = dataset
+      end
+
+      def recommended_dimensions
+
+        dim_objs = dimensions.map { |d| PublishMyData::DataCube::Dimension.new(d[:uri], self) }
+
+        sorted_dims = dim_objs.sort{ |x,y| y.size <=> x.size } # ordered by size desc
+
+        largest_dimension = sorted_dims.first
+        second_largest_dimension = sorted_dims[1]
+
+        locked_dims = {}
+
+        # other dims
+        (2...sorted_dims.length).each do |i|
+          dim = sorted_dims[i]
+          locked_dims[dim.uri] = dim.values.first[:uri]
+        end
+
+        {
+          rows_dimension: largest_dimension.uri,
+          columns_dimension: second_largest_dimension.uri,
+          locked_dimensions: locked_dims
+        }
+      end
+
+      # a collection of dimension properties for this cube
+      def dimensions
+
+        query = "PREFIX qb: <http://purl.org/linked-data/cube#>
+        PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+
+        SELECT DISTINCT ?uri ?label WHERE {
+          ?uri a qb:DimensionProperty .
+          OPTIONAL {
+            ?uri rdfs:label ?label .
+          }
+          GRAPH <#{dataset.data_graph_uri}> {
+            ?s ?uri ?o .
+          }
+        }"
+
+        uris_and_labels_only(Tripod::SparqlClient::Query.select(query))
+      end
+
+      # the (one and only) measure property for this cube.
+      def measure_property
+
+        query = "PREFIX qb: <http://purl.org/linked-data/cube#>
+        PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+
+        SELECT DISTINCT ?uri ?label WHERE {
+          ?uri a qb:MeasureProperty .
+          OPTIONAL {
+            ?uri rdfs:label ?label .
+          }
+          GRAPH <#{dataset.data_graph_uri}> {
+            ?s ?uri ?o .
+          }
+        }"
+
+        uris_and_labels_only(Tripod::SparqlClient::Query.select(query)).first
+      end
+
+      # For a given row and column dimension,
+      # and a hash of locked dimensions {dimension uri => value}
+      # Returns an arry of hashes for the page, with one row per item in the array.
+      def grid_observations(page, per_page, rows_dimension_uri, columns_dimension_uri, locked_dimensions={}, order_desc=false, order_by_column_uri=nil)
+        measure_property_uri = measure_property[:uri].to_s
+        sparql = grid_rows_observations_sparql(page, per_page, rows_dimension_uri, columns_dimension_uri, measure_property_uri, locked_dimensions, order_desc, order_by_column_uri)
+        results = Tripod::SparqlClient::Query.select(sparql)
+        grid_data(results, rows_dimension_uri, columns_dimension_uri)
+      end
+
+      # For a given row and column dimension,
+      # and a hash of locked dimensions {dimension uri => value}
+      def csv_observations(rows_dimension_uri, columns_dimension_uri, locked_dimensions={}, order_desc=false, order_by_column_uri=nil)
+        measure_property_uri = measure_property[:uri].to_s
+
+        page = 1
+        per_page = 5000
+        page_of_results = nil
+        results = []
+
+        while page == 1 || page_of_results.size == per_page
+          sparql = paged_observations_sparql(page, per_page, rows_dimension_uri, columns_dimension_uri, measure_property_uri, locked_dimensions, order_desc, order_by_column_uri)
+          page_of_results = Tripod::SparqlClient::Query.select(sparql)
+          results += page_of_results
+          page += 1
+        end
+
+        csv_data(results, rows_dimension_uri, columns_dimension_uri, locked_dimensions)
+      end
+
+      private
+
+      # convert the results in to a csv format.
+      def csv_data(sparql_results, rows_dimension_uri, columns_dimension_uri, locked_dimensions)
+
+        grid_data_results = grid_data(sparql_results, rows_dimension_uri, columns_dimension_uri)
+
+        # start the headers off with just the rows dimension label
+        header_uris = [nil, nil]
+        header_labels = [rows_dimension_uri, (Resource.find(rows_dimension_uri).label || rows_dimension_uri)]
+
+        columns = PublishMyData::DataCube::Dimension.new(columns_dimension_uri, self).values
+
+        columns.each do |col|
+          header_uris << col[:uri]
+          header_labels << (col[:label].present? ? col[:label] : col[:uri])
+        end
+
+        CSV.generate() do |csv|
+
+          csv << ["Generated by #{PublishMyData.local_domain}", Time.now.iso8601]
+          csv << [dataset.uri, dataset.title]
+
+          locked_dimensions.each_pair do |dimension_uri, dimension_value_uri|
+            dimension_res = Resource.find(dimension_uri) rescue nil
+            dimension_value_res = Resource.find(dimension_value_uri) rescue nil
+            csv << [((dimension_res && dimension_res.label) || dimension_uri),
+              ((dimension_value_res && dimension_value_res.label) || dimension_value_uri)]
+          end
+
+          csv << []
+
+          # add another row, with the column header URIs.
+          csv << header_uris
+          csv << header_labels
+
+          grid_data_results.each do |row|
+
+            row_array = []
+            row_array << ( row[rows_dimension_uri] ) # the row uri
+            row_array << ( row["rowlabel"].present? ? row["rowlabel"] : row[rows_dimension_uri] ) # the row label
+
+            header_uris[2,header_uris.length-1].each do |h|
+              if row[h]
+                row_array << row[h][:val]
+              else
+                row_array << nil
+              end
+            end
+
+            csv << row_array
+          end
+
+        end
+
+      end
+
+
+      # convert the sparql results into a grid format (array of hashes)
+      def grid_data(sparql_results, rows_dimension_uri, columns_dimension_uri)
+
+        rows_hash = {}
+
+        sparql_results.each do |result|
+          row_uri = result["row"]["value"]
+          column_uri = result["column"]["value"]
+          val = result["val"]["value"]
+          obs = result["obs"]["value"]
+          row_label = result["rowlabel"]["value"] if result["rowlabel"]
+          rows_hash[row_uri] ||= {}
+          rows_hash[row_uri]["rowlabel"] = row_label
+          rows_hash[row_uri][column_uri] = {val: val, obs: obs}
+        end
+
+        # we now have a fully popluated rows_hash.
+        # {
+        #  "row-1-uri": {"col-1-uri": {val: blah, obs: obs-uri}, "col-2-uri": {val: blah, obs: obs-uri},
+        #  "row-2-uri": ....
+        # }
+
+        # now build the actual rows to return
+        rows = []
+        rows_hash.each_pair do |row_uri, row_data|
+
+          # init the results row with just the row uri
+          row = {}
+          row[rows_dimension_uri] = row_uri
+
+          row_data.each_pair do |column_uri, column_value|
+            row[column_uri] = column_value
+          end
+
+          rows << row
+        end
+
+        rows
+      end
+
+      # build up a sparql query which gets a page observations (not based on grid rows).
+      # this is much quicker than the grid-paged version below.
+      def paged_observations_sparql(page, per_page, rows_dimension_uri, columns_dimension_uri, measure_property_uri, locked_dimensions={}, order_desc=false, order_by_column_uri=nil)
+        offset = (page-1) * per_page
+        order_dir = order_desc ? "DESC" : "ASC"
+
+        sparql = sparql_prefixes
+
+        sparql += "
+          SELECT ?row (?firstrowlabel as ?rowlabel) ?column ?obs ?val {
+
+            SELECT ?row (MIN(?rowlabel) as ?firstrowlabel) ?column ?obs ?val WHERE {
+
+              SELECT ?row ?rowlabel ?column ?obs ?val WHERE {
+                GRAPH <#{dataset.data_graph_uri.to_s}> {
+                  ?obs <#{rows_dimension_uri.to_s}> ?row .
+                  ?obs <#{columns_dimension_uri.to_s}> ?column .
+                  ?obs <#{measure_property_uri.to_s}> ?val ."
+
+              locked_dimensions.each_pair do |dimension_uri, dimension_value|
+                sparql += "
+                  ?obs <#{dimension_uri.to_s}> <#{dimension_value.to_s}> . "
+              end
+              sparql += "
+                }
+                "
+                if order_by_column_uri
+                  sparql += optional_column_ordering_clauses(order_by_column_uri, rows_dimension_uri, columns_dimension_uri, measure_property_uri, locked_dimensions)
+                end
+
+              sparql += "
+                OPTIONAL {
+                  ?row rdfs:label ?rowlabel .
+                }
+              }
+
+            }
+            GROUP BY ?row ?column ?obs ?val
+          }
+          "
+        if order_by_column_uri
+          sparql += "
+            ORDER BY #{order_dir}(?val) "
+        else
+          sparql += "
+            ORDER BY #{order_dir}(?firstrowlabel)" # order by the row labels by default
+        end
+
+        sparql += "LIMIT #{per_page} OFFSET #{offset}"
+
+      end
+
+      # build up a sparql query which gets the observations for a set of rows of the cube grid
+      def grid_rows_observations_sparql(page, per_page, rows_dimension_uri, columns_dimension_uri, measure_property_uri, locked_dimensions={}, order_desc=false, order_by_column_uri=nil)
+
+        offset = (page-1) * per_page
+        order_dir = order_desc ? "DESC" : "ASC"
+
+        sparql = sparql_prefixes
+        sparql += "
+          SELECT ?row (?firstrowlabel as ?rowlabel) ?column (?o2 as ?obs) (?val2 as ?val)
+          WHERE {
+            {
+              SELECT ?row (MIN(?rowlabel) as ?firstrowlabel) WHERE {
+              {
+                SELECT DISTINCT ?row
+                WHERE {
+                  GRAPH <#{dataset.data_graph_uri.to_s}> {
+                    ?o <#{rows_dimension_uri.to_s}> ?row . "
+
+          # add the locked dimensions here so that we get the right limited set of rows (the ref dimensions etc might change between years etc).
+          locked_dimensions.each_pair do |dimension_uri, dimension_value|
+            sparql += "
+                  ?o <#{dimension_uri.to_s}> <#{dimension_value.to_s}> . "
+          end
+
+          sparql += " }
+                }
+              }"
+
+        # only need to add these clauses if ordering by a column
+        if order_by_column_uri
+          sparql += optional_column_ordering_clauses(order_by_column_uri, rows_dimension_uri, columns_dimension_uri, measure_property_uri, locked_dimensions)
+        end
+
+        # put the row label constraint in its own optional block
+        sparql += "
+              OPTIONAL {
+                ?row rdfs:label ?rowlabel .
+              }
+            }
+
+            # group by rows so we can get the first label
+            GROUP BY ?row
+        "
+
+        if order_by_column_uri
+          sparql += "
+            ?val # this is an extra group by
+            ORDER BY #{order_dir}(?val) "
+        else
+          sparql += "
+            ORDER BY #{order_dir}(?firstrowlabel) " # order by the row labels by default
+        end
+
+        sparql += "
+            LIMIT #{per_page} OFFSET #{offset}
+          }
+
+          GRAPH <#{dataset.data_graph_uri.to_s}> {
+            ?o2 <#{rows_dimension_uri.to_s}> ?row .
+            ?o2 <#{columns_dimension_uri.to_s}> ?column .
+            ?o2 <#{measure_property_uri.to_s}> ?val2 .
+          "
+
+        locked_dimensions.each_pair do |dimension_uri, dimension_value|
+          sparql += "
+            ?o2 <#{dimension_uri.to_s}> <#{dimension_value.to_s}> . "
+        end
+
+        sparql += "
+            }
+          } "
+        sparql
+      end
+
+      def sparql_prefixes
+        "PREFIX qb: <http://purl.org/linked-data/cube#>
+          PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+        "
+      end
+
+      def optional_column_ordering_clauses(order_by_column_uri, rows_dimension_uri, columns_dimension_uri, measure_property_uri, locked_dimensions)
+        sparql = "
+          OPTIONAL {
+            GRAPH <#{dataset.data_graph_uri.to_s}> {
+              ?o3 <#{rows_dimension_uri.to_s}> ?row .
+              ?o3 <#{columns_dimension_uri.to_s}> <#{order_by_column_uri.to_s}> .
+              ?o3 <#{measure_property_uri.to_s}> ?val . "
+
+        locked_dimensions.each_pair do |dimension_uri, dimension_value|
+          sparql += "
+            ?o3 <#{dimension_uri.to_s}> <#{dimension_value.to_s}> . "
+        end
+
+        sparql += " }
+          } "
+
+        return sparql
+      end
+
+    end
+
+  end
+end

data/app/models/publish_my_data/data_cube/dimension.rb

@@ -0,0 +1,58 @@
+module PublishMyData
+  module DataCube
+
+    class Dimension
+
+      include PublishMyData::CubeResults
+
+      attr_reader :uri
+      attr_reader :cube
+
+      PAGE_SIZE = 5000
+
+      def initialize(uri, cube)
+        @uri = uri
+        @cube = cube
+      end
+
+      # get all the possible values (uris and labels) for this dimension in the dataset passed in.
+      # paginates internally if necessary.
+      def values
+        sparql = values_sparql(labels:true)
+        results = loop_and_page_sparql_query(sparql) # by default this tries 5000 rows at a time.
+        uris_and_labels_only(results)
+      end
+
+      def size
+        # don't get labels and don't paginate.
+        sparql = values_sparql({labels:false})
+        sq = Tripod::SparqlQuery.new(sparql)
+        count_sparql = sq.as_count_query_str
+        result = Tripod::SparqlClient::Query.select(count_sparql)
+        result[0]["tripod_count_var"]["value"].to_i
+      end
+
+      private
+
+      def values_sparql(opts={})
+        labels = opts[:labels]
+
+        sparql = "PREFIX qb: <http://purl.org/linked-data/cube#>
+        PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
+        SELECT DISTINCT ?uri #{labels ? '?label' : ""} WHERE {
+          GRAPH <#{cube.dataset.data_graph_uri.to_s}> {
+            ?obs qb:dataSet <#{cube.dataset.uri.to_s}> .
+            ?obs <#{self.uri.to_s}> ?uri .
+          }
+        "
+        sparql += "OPTIONAL { ?uri rdfs:label ?label . } " if labels
+        sparql += " } "
+        sparql += " ORDER BY #{labels ? '?label' : ''} ?uri"
+        sparql
+      end
+
+
+
+    end
+  end
+end

data/app/models/publish_my_data/vocabulary.rb

@@ -4,10 +4,6 @@ module PublishMyData
     include Tripod::Resource
     include AllFeatures
 
-    def self.find_by_data_dump(data_dump_uri)
-      all.where("?uri <#{RDF::VOID.dataDump}> <#{data_dump_uri}>").first
-    end
-
     # override
     def slug
       uri_hash(self.uri)

data/app/views/layouts/publish_my_data/application.html.haml

@@ -0,0 +1,18 @@
+!!! 5
+%html
+  %head
+    %title
+      = render 'publish_my_data/shared/meta_title'
+    = yield :page_description
+    = javascript_include_tag :publish_my_data
+    = stylesheet_link_tag :publish_my_data
+  %body
+    /[if lt IE 8]
+      = render 'publish_my_data/shared/browser_warning'
+    %header
+      %nav
+        = render 'publish_my_data/shared/logo'
+        = yield :crumbs
+    %main= yield
+    = render 'publish_my_data/shared/footer'
+    = render 'publish_my_data/shared/google_analytics'

data/app/views/layouts/publish_my_data/error.html.haml

@@ -0,0 +1,21 @@
+- # error layout: by default this is the same as application layout
+- # but it's a separate layout for easier custom error pages in your app.
+
+!!! 5
+%html
+  %head
+    %title
+      = render 'publish_my_data/shared/meta_title'
+    = yield :page_description
+    = javascript_include_tag :publish_my_data
+    = stylesheet_link_tag :publish_my_data
+  %body
+    /[if lt IE 8]
+      = render 'publish_my_data/shared/browser_warning'
+    %header
+      %nav
+        = render 'publish_my_data/shared/logo'
+        = yield :crumbs
+    %main= yield
+    = render 'publish_my_data/shared/footer'
+    = render 'publish_my_data/shared/google_analytics'

data/app/views/publish_my_data/classes/show.html.haml

@@ -0,0 +1,28 @@
+- page_title "Class: #{ontology_class.label || ontology_class.uri}"
+
+- content_for :crumbs do
+  %ol.breadcrumb
+    %li= link_to_home
+    - if ontology_class.defined_by_ontology
+      %li= link_to "Ontology: #{ontology_class.defined_by_ontology.label || ontology_class.defined_by_ontology.uri}", resource_path_from_uri(ontology_class.defined_by_ontology.uri)
+    %li Class
+
+%article#resource
+  %header
+    %h3 Class
+    %h1= ontology_class.label || ontology_class.uri
+    %h3 URI
+    %h2.click-to-select= ontology_class.uri
+
+  = render  'publish_my_data/resources/resource_data', resource: ontology_class
+
+  %footer.alternative-formats
+    :markdown
+      This **class** is available as:
+    %ul
+      %li HTML
+      %li= link_to("JSON", show_resource_path(:uri => ontology_class.uri, :format => 'json'))
+      %li= link_to("RDF/XML", show_resource_path(:uri => ontology_class.uri, :format => 'rdf'))
+      %li= link_to("Turtle", show_resource_path(:uri => ontology_class.uri, :format => 'ttl'))
+      %li= link_to("N-triples", show_resource_path(:uri => ontology_class.uri, :format => 'nt'))
+

data/app/views/publish_my_data/concept_schemes/show.html.haml

@@ -0,0 +1,79 @@
+- page_title("Concept Scheme: #{concept_scheme.label || concept_scheme.uri}")
+
+- content_for :crumbs do
+  %ol.breadcrumb
+    %li= link_to_home
+    %li= link_to 'Concept Scheme', resource_path_from_uri(concept_scheme.uri)
+
+- if concept_scheme.deprecated?
+  = render partial: 'publish_my_data/shared/deprecation_notice', locals: {resource: dataset}
+
+%article#concept-scheme
+  %header
+    %h3= concept_scheme.local? ? 'Concept Scheme' : 'External Concept Scheme'
+    %h1= concept_scheme.label || concept_scheme.uri
+    %h3 URI
+    %h2.click-to-select= concept_scheme.uri
+
+  %section.metadata
+    .primary
+      - if concept_scheme.comment.present?
+        %h3 Description
+        %p= auto_link concept_scheme.comment.to_s
+      - if concept_scheme.publisher.present?
+        %h3 Publisher
+        %p= resource_uri_or_label(concept_scheme, concept_scheme.publisher)
+      - if concept_scheme.license.present?
+        %h3 Licence
+        %p= resource_uri_or_label(concept_scheme, concept_scheme.license)
+      - if concept_scheme.contact_email.present?
+        %h3 Contact
+        %p= auto_link concept_scheme.contact_email.to_s.gsub('mailto:','')
+      - if concept_scheme.description.present?
+        %h3 Further Information
+        .js-only
+          #further-information.truncated
+            :markdown
+              #{concept_scheme.description.to_s}
+            .fade-out
+        %p
+          %a#read-more(href="#") Read more
+        %noscript
+          :markdown
+            #{concept_scheme.description.to_s}
+          :css
+            .js-only { display: none; }
+        :javascript
+          $("a#read-more").click(function(e) {
+            e.preventDefault();
+            $("#further-information").removeClass("truncated");
+            $("#further-information .fade-out").hide();
+            $("a#read-more").closest('p').hide();
+          });
+    .secondary
+      - if concept_scheme.issued.present?
+        %h3 Issued
+        %p= formatted_date(concept_scheme.issued.to_s)
+      - if concept_scheme.modified.present?
+        %h3 Modified
+        %p= formatted_date(concept_scheme.modified.to_s)
+      - if concept_scheme.tags.try(:any?)
+        %h3 Tags
+        %p= concept_scheme.tags.join(', ')
+
+  %ul.tab-links
+    %li.active
+      %a(href="#concepts" data-toggle="tab") Concepts
+    %li
+      %a(href="#linked-data" data-toggle="tab") Linked Data & SPARQL
+
+  .tab-content
+    %section#concepts.active
+      .content
+        %ul
+          - concept_scheme.concepts.each do |concept|
+            %li
+              %h3= resource_uri_or_label(concept, concept.uri)
+              %h4.click-to-select= concept.uri
+              %p= concept.comment
+    = render 'publish_my_data/resources/sparql_section', resource: concept_scheme

data/app/views/publish_my_data/concepts/show.html.haml

@@ -0,0 +1,28 @@
+- page_title "Concept #{concept.label || concept.uri}"
+
+- content_for :crumbs do
+  %ol.breadcrumb
+    %li= link_to_home
+    - if concept.concept_scheme
+      %li= link_to "Concept Scheme: #{concept.concept_scheme.label || concept.concept_scheme.uri}", resource_path_from_uri(concept.concept_scheme.uri)
+    %li Concept
+
+%article#resource
+  %header
+    %h3 Concept
+    %h1= concept.label || concept.uri
+    %h3 URI
+    %h2.click-to-select= concept.uri
+
+  = render  'publish_my_data/resources/resource_data', resource: concept
+
+  %footer.alternative-formats
+    :markdown
+      This **concept** is available as:
+    %ul
+      %li HTML
+      %li= link_to("JSON", show_resource_path(:uri => concept.uri, :format => 'json'))
+      %li= link_to("RDF/XML", show_resource_path(:uri => concept.uri, :format => 'rdf'))
+      %li= link_to("Turtle", show_resource_path(:uri => concept.uri, :format => 'ttl'))
+      %li= link_to("N-triples", show_resource_path(:uri => concept.uri, :format => 'nt'))
+

data/app/views/publish_my_data/data_cube/_controls.html.haml

@@ -0,0 +1,19 @@
+#dimensions-controls
+  %form(style="margin:0; display:none")
+    .rows-and-columns
+      %fieldset.rows-and-columns
+        %legend Row and Column Headings
+
+        .field
+          %label Column Headings
+          %div#columns-dimension-container.picker
+
+        .field
+          %label Rows (values in first column)
+          %div#rows-dimension-container.picker
+
+    .other-options
+      %fieldset
+        %legend Other Options
+        .locked-dimensions
+          No other options for this dataset

data/app/views/publish_my_data/data_cube/_grid.html.haml

@@ -0,0 +1,37 @@
+.content
+  .grid-intro
+    :markdown
+      This dataset contains multidimensional data (a _data cube_). This grid allows you to view two dimensions at a time.
+
+      Use the drop-downs below the grid to choose which dimensions to use for the rows and columns.
+
+  .grid-header
+    .grid-title
+      %h3#cube-grid-title
+         
+    .grid-status
+      .status-value
+      = image_tag("publish_my_data/small-spinner.gif", alt: "busy", style: "display:none", class: "busy")
+
+  #data-cube-grid.data-grid
+  .grid-footer
+    .footer-content
+
+.controls-wrapper
+  = render 'publish_my_data/data_cube/controls'
+
+.content
+  %h3 Help
+  .docs(style="padding-top:7px")
+    %ul
+      %li
+        To view the information held in our database for a column or individual cell, just click the blue links in the grid.
+      %li
+        You can re-order a column's contents by clicking on its header (anywhere except the links themselves), or resize it by dragging the column boundaries.
+      %li
+        The entire contents of grid (based on the currently selected options and ordering) can be downloaded in CSV format via the link in the grid footer.
+
+  :javascript
+    $(function() {
+      new Swirrl.DatasetCubeGrid(window.location.host, '#{dataset.slug}', "#{dataset.title}", "#data-cube-grid");
+    });