data_kitten 1.2.0 → 1.3.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    NDc5ZTUwMDA5OGJjMjg3YzI4MThmZWYzNTQ1NTkyNTNlN2JhZWU5Nw==
+    MTU4NmQ1MmU2YjJhY2U2NmVjOWE5NTBhOGM2YjNhNGQzMWIxYTU4MQ==
   data.tar.gz: !binary |-
-    ZTkzNGQxYWIwOWU5NjliZThmZjE0Y2FkMWE1NmFhYTkwMTY2MGJhOQ==
+    ZjNhNWU4NzNlZjI4ODU3ZTRkYTgxZmY4MWM0NTA3OTQyNDNmMmJjMg==
 SHA512:
   metadata.gz: !binary |-
-    MGM1NjQ2ZDUzYzJiYjY5NzY1OTk1YzI0ZDZhZDQ3ZGIwZjdhNDVmYzRjNzM4
-    ODc2MTUyYzFiOGNmNmNjNzExZjFmMTllZmFiNDMyZThiY2VmYjEyYThkZWMz
-    ZmU2NjUyMDQ2MjVkYTNlYmNhNDBhOGYzYzE2Nzk3NjFmYTAyMTY=
+    ZjY0ODMyZmZjODMxNmU4NmNhMmI4N2YxMWMzMThhNGFlMTAxZmQ0ODA1OTcz
+    MjcwOWIwMmYyMmRiNDlmNTEyN2ExMWE4N2E4NWJlMDFlNjI3NDU4ZmZjZWFj
+    Yjk2MWZiZGJmODE5ZDVjMWQ0OTY2ODI4MWRiMzVlZmE5NjM0OGM=
   data.tar.gz: !binary |-
-    ZWJhOTU2YTNkMmY3MzhiZjllNTY4YmY1MDFjNGZlZmY1MjIwODQ4ZDA5OGIz
-    N2YwMjk4YzM5ZWUyM2VhODJjZjgzM2QzY2JjYjA5ZDFhMTUzM2U0MGU4ZDYz
-    NjUxN2ExYTg0MTJlZTJkN2U2MTc1ZDZjMjBjOWQ5NzU3MWI1MGY=
+    MjY5NDZiZjliNThjZTk5NTQ4YzZlM2M4OGFiOGYzYjFjZDhmYzQ4NmRlMjJm
+    ZmE4MTMxZTMyZmQzNjBhODEyODZhODA3ZjIyMDUzNDFmMWJiMWRhNTBlMTU3
+    MTM2MTU4ZGY4YmZlMjcyM2VkYmM5Y2Q3NzY3YWJmNmNlODdlZTI=

data/lib/data_kitten/dataset.rb

@@ -49,7 +49,11 @@ module DataKitten
     def url
       @access_url.to_s
     end
-  
+
+    def source
+      @access_url.as_json if @access_url.ok?
+    end
+
     # Can metadata be loaded for this Dataset?
     #
     # @return [Boolean] true if metadata can be loaded, false if it's 
@@ -75,7 +79,15 @@ module DataKitten
     def host
       nil
     end
-    
+
+    # A unique identifier of the dataset.
+    #
+    # @return [String] the identifier of the dataset
+    #
+    def identifier
+      nil
+    end
+
     # The human-readable title of the dataset.
     #
     # @return [String] the title of the dataset.
@@ -126,6 +138,13 @@ module DataKitten
     def modified
       nil
     end
+
+    # A web page that can be used to gain access to the dataset, its distributions and/or additional information.
+    #
+    # @return [String] The URL to the dataset
+    def landing_page
+      nil
+    end
     
     # The temporal coverage of the dataset
     #
@@ -191,7 +210,21 @@ module DataKitten
     def contributors
       []
     end
-    
+
+    # The language of the dataset.
+    #
+    # @return [String] the language of the dataset
+    def language
+      nil
+    end
+
+    # The main category the dataset belongs to.
+    #
+    # @return [String]
+    def theme
+      nil
+    end
+
     # Has the data been crowdsourced?
     #
     # @return [Boolean] Whether the data has been crowdsourced or not.
@@ -229,5 +262,12 @@ module DataKitten
       []
     end
 
+    # Spatial coverage of the dataset
+    #
+    # @return [GeoJSON Geometry] A GeoJSON geometry object of the spatial coverage
+    def spatial
+      nil
+    end
+
   end
 end

data/lib/data_kitten/distribution.rb

@@ -14,8 +14,11 @@ module DataKitten
     # @!attribute access_url
     #   @return [String] a URL to access the distribution.
     attr_accessor :access_url
-    alias_method :uri, :access_url
-    alias_method :download_url, :access_url
+
+    # @!attribute download_url
+    #   @return [String] a URL to the file of the distribution.
+    attr_accessor :download_url
+    alias_method :uri, :download_url
 
     # @!attribute path
     #   @return [String] the path of the distribution within the source, if appropriate
@@ -29,6 +32,22 @@ module DataKitten
     #   @return [String] a textual description
     attr_accessor :description
 
+    # @!attribute issued
+    #   @return [Date] date created
+    attr_accessor :issued
+
+    # @!attribute modified
+    #   @return [Date] date modified
+    attr_accessor :modified
+
+    # @!attribute byte_size
+    #   @return [Integer] size of file in bytes
+    attr_accessor :byte_size
+
+    # @!attribute media_type
+    #   @return [String] the IANA media type (MIME type) of the distribution
+    attr_accessor :media_type
+
     # @!attribute schema
     #   @return [Hash] a hash representing the schema of the data within the distribution. Will
     #                  change to a more structured object later.
@@ -65,7 +84,7 @@ module DataKitten
         @schema = r['schema']
         # Get path
         @path = r['path']
-        @access_url = r['url']
+        @download_url = r['url']
         # Set title
         @title = @path || @uri
       elsif r = options[:dcat_resource]
@@ -73,13 +92,18 @@ module DataKitten
         @description = r[:title]
         @access_url  = r[:accessURL]
       elsif r = options[:ckan_resource]
-        @title       = r[:title]
-        @description = r[:title]
-        @access_url  = r[:accessURL]
-        @extension   = r[:format]
+        @title        = r[:title]
+        @description  = r[:title]
+        @issued       = r[:issued]
+        @modified     = r[:modified]
+        @access_url   = r[:accessURL]
+        @download_url = r[:downloadURL]
+        @byte_size    = r[:byteSize]
+        @media_type   = r[:mediaType]
+        @extension    = r[:format]
         # Load HTTP Response for further use
         @format = r[:format] ? DistributionFormat.new(self) : nil
-    end
+      end
       # Set default CSV dialect
       @dialect ||= {
         "delimiter" => ","
@@ -112,7 +136,7 @@ module DataKitten
     #
     # @return [Boolean] whether the HTTP response returns a success code or not
     def exists?
-      if @access_url
+      if @download_url
         http_head.response_code != 404
       end
     end
@@ -124,8 +148,8 @@ module DataKitten
       @data ||= begin
         if @path
           datafile = @dataset.send(:load_file, @path)
-        elsif @access_url
-          datafile = RestClient.get @access_url rescue nil
+        elsif @download_url
+          datafile = RestClient.get @download_url rescue nil
         end
         if datafile
           case format.extension
@@ -147,9 +171,9 @@ module DataKitten
     end
 
     def http_head
-      if @access_url
+      if @download_url
         @http_head ||= begin
-          Curl::Easy.http_head(@access_url) do |c|
+          Curl::Easy.http_head(@download_url) do |c|
             c.follow_location = true
             c.useragent = "curb"
           end

data/lib/data_kitten/distribution_format.rb

@@ -21,6 +21,7 @@ module DataKitten
       @@formats ||= {
         csv:     { structured:  true, open:  true },
         xls:     { structured:  true, open: false },
+        xlsx:    { structured:  true, open:  true },
         rdf:     { structured:  true, open:  true },
         xml:     { structured:  true, open:  true },
         wms:     { structured:  true, open:  true },
@@ -33,6 +34,7 @@ module DataKitten
         sparql:  { structured:  true, open:  true },
         kml:     { structured:  true, open:  true },
         georss:  { structured:  true, open:  true },
+        geojson: { structured:  true, open:  true },
         shp:     { structured:  true, open:  true },
         html:    { structured: false, open:  true },
         doc:     { structured: false, open:  false },

data/lib/data_kitten/license.rb

@@ -3,6 +3,23 @@ module DataKitten
   # A license for a {Dataset} or {Distribution}
   #
   class License
+    
+    LICENSES = {
+      /opendatacommons.org.*\/by(\/|$)/ => "odc-by",
+      /opendatacommons.org.*\/odbl(\/|$)/ => "odc-odbl",
+      /opendatacommons.org.*\/pddl(\/|$)/ => "odc-pddl",
+      /opendefinition.org.*\/odc-by(\/|$)/ => "odc-by",
+      /opendefinition.org.*\/odc-pddl(\/|$)/ => "odc-pddl",
+      /opendefinition.org.*\/cc-zero(\/|$)/ => "cc-zero",
+      /opendefinition.org.*\/cc-by(\/|$)/ => "cc-by",
+      /opendefinition.org.*\/cc-by-sa(\/|$)/ => "cc-by-sa",
+      /opendefinition.org.*\/gfdl(\/|$)/ => "gfdl",
+      /creativecommons.org.*\/zero(\/|$)/ => "cc-zero",
+      /creativecommons.org.*\/by-sa(\/|$)/ => "cc-by-sa",
+      /creativecommons.org.*\/by(\/|$)/ => "cc-by",
+      /(data|nationalarchives).gov.uk.*\/open-government-licence(\/|$)/ => "ogl-uk",
+      /usa.gov\/publicdomain(\/|$)/ => "us-pd"
+    }
 
     # @!attribute is
     #   @return [String] a short ID that identifies the license.
@@ -19,6 +36,10 @@ module DataKitten
     # @!attribute type
     #   @return [String] the type of information this license applies to. Could be +:data+ or +:content+.
     attr_accessor :type
+    
+    # @!attribute abbr
+    #   @return [String] the license abbreviation
+    attr_accessor :abbr
 
     # Create a new License object.
     #
@@ -32,8 +53,14 @@ module DataKitten
       @name = options[:name]
       @uri = options[:uri]
       @type = options[:type]
+      @abbr = get_license_abbr(@uri) if @uri
+    end
+    
+    def get_license_abbr(uri)
+      license = LICENSES.find { |regex, abbr| uri =~ regex }
+      license.last if license
     end
 
   end  
 
-end
+end

data/lib/data_kitten/publishing_formats/ckan.rb

@@ -1,3 +1,5 @@
+require 'data_kitten/utils/guessable_lookup.rb'
+
 module DataKitten
 
   module PublishingFormats
@@ -26,6 +28,7 @@ module DataKitten
           @@id = result["result"]["id"] rescue result["id"]
         end
         @@metadata = JSON.parse RestClient.get "#{uri.scheme}://#{uri.host}/api/rest/package/#{@@id}"
+        @@metadata.extend(GuessableLookup)
       rescue
         false
       end
@@ -43,14 +46,32 @@ module DataKitten
       #
       # @see Dataset#data_title
       def data_title
-        metadata["title"] rescue nil
+        metadata.lookup("title")
       end
 
       # A brief description of the dataset
       #
       # @see Dataset#description
       def description
-        metadata["notes"] rescue nil
+        metadata.lookup("notes") || metadata.lookup("description")
+      rescue 
+        nil
+      end
+
+      # An identifier for the dataset
+      #
+      # @see Dataset#identifier
+      def identifier
+        metadata.lookup("name") || @@id
+      end
+
+      # A web page which can be used to gain access to the dataset
+      #
+      # @see Dataset#landing_page
+      def landing_page
+        metadata.lookup("extras", "landing_page") ||
+        metadata.lookup("url") ||
+        metadata.lookup("ckan_url")
       end
 
       # Keywords for the dataset
@@ -58,7 +79,7 @@ module DataKitten
       # @see Dataset#keywords
       def keywords
         keywords = []
-        metadata["tags"].each do |tag|
+        metadata.lookup("tags").each do |tag|
           keywords << tag
         end
         return keywords
@@ -70,7 +91,7 @@ module DataKitten
       #
       # @see Dataset#publishers
       def publishers
-        id = metadata['organization']['id'] || metadata['groups'][0]
+        id = metadata.lookup('organization', 'id') || metadata.lookup('groups', 0)
         fetch_publisher(id)
       rescue
         []
@@ -88,10 +109,9 @@ module DataKitten
       #
       # @see Dataset#licenses
       def licenses
-        extras = metadata["extras"] || {}
-        id = metadata["license_id"]
-        uri = metadata["license_url"] || extras["licence_url"]
-        name = metadata["license_title"] || extras["licence_url_title"]
+        id = metadata.lookup("license_id")
+        uri = metadata.lookup("license_url") || metadata.lookup("extras", "licence_url")
+        name = metadata.lookup("license_title") || metadata.lookup("extras", "licence_url_title")
         if [id, uri, name].any?
           [License.new(:id => id, :uri => uri, :name => name)]
         else
@@ -104,12 +124,17 @@ module DataKitten
       # @see Dataset#distributions
       def distributions
         distributions = []
-        metadata["resources"].each do |resource|
+        metadata.lookup("resources").each do |resource|
           distribution = {
             :title => resource["description"],
-            :accessURL => resource["url"],
-            :format => resource["format"]
+            :accessURL => landing_page,
+            :downloadURL => resource["url"],
+            :format => resource["format"],
+            :mediaType => resource["mimetype"] || resource["content_type"],
           }
+          distribution[:issued] = Date.parse(resource["created"]) rescue nil
+          distribution[:modified] = Date.parse(resource["last_modified"] || resource["revision_timestamp"]) rescue nil
+          distribution[:byteSize] = Integer(resource["size"]) rescue nil
           distributions << Distribution.new(self, ckan_resource: distribution)
         end
         return distributions
@@ -121,32 +146,68 @@ module DataKitten
       #
       # @see Dataset#update_frequency
       def update_frequency
-        metadata["extras"]["update_frequency"] || metadata["extras"]["frequency-of-update"] rescue nil
+        metadata.lookup("extras", "update_frequency") ||
+        metadata.lookup("extras", "frequency-of-update") ||
+        metadata.lookup("extras", "accrual_periodicity")
+      rescue
+        nil
       end
 
       # Date the dataset was released
       #
       # @see Dataset#issued
       def issued
-        Date.parse metadata["metadata_created"] rescue nil
+        Date.parse metadata.lookup("metadata_created") rescue nil
       end
 
       # Date the dataset was modified
       #
       # @see Dataset#modified
       def modified
-        Date.parse metadata["metadata_modified"] rescue nil
+        Date.parse metadata.lookup("metadata_modified") rescue nil
       end
 
       # The temporal coverage of the dataset
       #
       # @see Dataset#temporal
       def temporal
-        start_date = Date.parse metadata["extras"]["temporal_coverage-from"] rescue nil
-        end_date = Date.parse metadata["extras"]["temporal_coverage-to"] rescue nil
+        from = metadata.lookup("extras", "temporal_coverage-from") ||
+               metadata.lookup("extras", "temporal-extent-begin")
+        to = metadata.lookup("extras", "temporal_coverage-to") ||
+             metadata.lookup("extras", "temporal-extent-end")
+        start_date = Date.parse from rescue nil
+        end_date = Date.parse to rescue nil
         Temporal.new(:start => start_date, :end => end_date)
       end
 
+      # The language of the dataset
+      #
+      # @see Dataset#language
+      def language
+        metadata.lookup("language") ||
+        metadata.lookup("metadata_language") ||
+        metadata.lookup("extras", "metadata_language") ||
+        metadata.lookup("extras", "language", 0) ||
+        metadata.lookup("extras", "language")
+      end
+
+      # The main category of the dataset
+      #
+      # @see Dataset#theme
+      def theme
+        metadata.lookup("extras", "theme", 0) ||
+        metadata.lookup("extras", "theme-primary") ||
+        metadata.lookup("groups", 0, "name") ||
+        metadata.lookup("groups", 0)
+      end
+
+      # Spatial coverage of the dataset
+      #
+      # @see Dataset#spatial
+      def spatial
+        extract_spatial || extract_bbox
+      end
+
       private
 
       def metadata
@@ -161,17 +222,45 @@ module DataKitten
         extra
       end
 
+      def extract_spatial
+        geometry = JSON.parse metadata.lookup("extras", "spatial")
+        return geometry if !geometry["type"].nil?
+      rescue
+        nil
+      end
+      
+      def extract_bbox
+        west = Float(metadata.lookup("extras", "bbox-west-long"))
+        east = Float(metadata.lookup("extras", "bbox-east-long"))
+        north = Float(metadata.lookup("extras", "bbox-north-lat"))
+        south = Float(metadata.lookup("extras", "bbox-south-lat"))
+
+        { "type" => "Polygon", "coordinates" => [
+          [
+            [west, north],
+            [east, north],
+            [east, south],
+            [west, south],
+            [west, north]
+          ]
+        ] }
+      rescue
+        nil
+      end
+
       def fetch_publisher(id)
         uri = parsed_uri
         [
-          "#{uri.scheme}://#{uri.host}/api/rest/group/#{id}",
+          "#{uri.scheme}://#{uri.host}/api/3/action/organization_show?id=#{id}",
           "#{uri.scheme}://#{uri.host}/api/3/action/group_show?id=#{id}",
-          "#{uri.scheme}://#{uri.host}/api/3/action/organization_show?id=#{id}"
+          "#{uri.scheme}://#{uri.host}/api/rest/group/#{id}"
         ].each do |uri|
           begin
             @group = JSON.parse RestClient.get uri
             break
-          rescue RestClient::ResourceNotFound
+          rescue
+            # FakeWeb raises FakeWeb::NetConnectNotAllowedError, whereas 
+            # RestClient raises RestClient::ResourceNotFound in the "real world".
             nil
           end
         end
@@ -190,8 +279,8 @@ module DataKitten
       end
 
       def extract_agent(name_field, email_field)
-        name = metadata[name_field]
-        email = metadata[email_field]
+        name = metadata.lookup(name_field)
+        email = metadata.lookup(email_field)
         if [name, email].any?
           [Agent.new(name: name, mbox: email)]
         else

data/lib/data_kitten/utils/guessable_lookup.rb

@@ -0,0 +1,38 @@
+module GuessableLookup
+  
+  def lookup(*path)
+    data = self
+    path.each { |key| data = guess_key(data, key) }
+    data
+  rescue
+    nil
+  end
+  
+  private
+    
+  # Guesses which key you want from a hash and returns the value of it.
+  #
+  # It returns the value of the original key if it exists in the hash, otherwise 
+  # tries to find a similar key, and if it fails it returns nil. 
+  # Similar keys are ones which use '_', '-' or '' as word separators & are 
+  # case-insensitive.
+  # 
+  # @example 
+  #   guess_key({:a_key => true}, 'a_key')  # => true
+  #   guess_key({:aKey => true}, 'a_key')   # => true
+  #   guess_key({"a-KEY" => true}, 'a_key') # => true
+  #
+  # @param data [Hash] 
+  # @param key [String] The desired key
+  # @return The value of the guessed key
+  #
+  def guess_key(data, key)
+    return data[key] if key.is_a?(Fixnum) || data.keys.include?(key)
+    likeKey = key.gsub(/[\_\-]/, "[\_\-]?")
+    key = data.keys.select { |k| k =~ /^#{likeKey}$/i }.first
+    data[key]
+  rescue
+    nil
+  end
+  
+end

data/lib/data_kitten/version.rb

@@ -1,3 +1,3 @@
 module DataKitten
-  VERSION = "1.2.0"
+  VERSION = "1.3.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: data_kitten
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.0
 platform: ruby
 authors:
 - James Smith
@@ -214,6 +214,7 @@ files:
 - lib/data_kitten/rights.rb
 - lib/data_kitten/source.rb
 - lib/data_kitten/temporal.rb
+- lib/data_kitten/utils/guessable_lookup.rb
 - lib/data_kitten/version.rb
 homepage: http://github.com/data-kitten
 licenses: