data_kitten 1.0.0

data/lib/data_kitten/distribution.rb

@@ -0,0 +1,156 @@
+module DataKitten
+
+  # A specific available form of a dataset, such as a CSV file, an API, or an RSS feed.
+  #
+  # Based on {http://www.w3.org/TR/vocab-dcat/#class-distribution dcat:Distribution}, but 
+  # with useful aliases for other vocabularies.
+  #
+  class Distribution
+    
+    # @!attribute format
+    #   @return [DistributionFormat] the file format of the distribution.
+    attr_accessor :format
+
+    # @!attribute access_url
+    #   @return [String] a URL to access the distribution.
+    attr_accessor :access_url
+    alias_method :uri, :access_url
+    alias_method :download_url, :access_url
+
+    # @!attribute path
+    #   @return [String] the path of the distribution within the source, if appropriate
+    attr_accessor :path
+
+    # @!attribute title
+    #   @return [String] a short title, unique within the dataset
+    attr_accessor :title
+
+    # @!attribute description
+    #   @return [String] a textual description
+    attr_accessor :description
+
+    # @!attribute schema
+    #   @return [Hash] a hash representing the schema of the data within the distribution. Will
+    #                  change to a more structured object later.
+    attr_accessor :schema
+
+    # Create a new Distribution. Currently only loads from Datapackage +resource+ hashes.
+    #
+    # @param dataset [Dataset] the {Dataset} that this is a part of.
+    # @param options [Hash] A set of options with which to initialise the distribution.
+    # @option options [String] :datapackage_resource the +resource+ section of a Datapackage 
+    #                                                representation to load information from.
+    def initialize(dataset, options) 
+      # Store dataset
+      @dataset = dataset
+      # Parse datapackage
+      if r = options[:datapackage_resource]
+        # Load basics
+        @description = r['description']
+        # Load HTTP Response for further use
+        if r['url']
+          @response = Curl::Easy.http_head(r['url'])
+        end
+        # Work out format
+        @format = begin
+          extension = r['format']
+          if extension.nil?
+            extension = r['path'].is_a?(String) ? r['path'].split('.').last.upcase : nil
+          end
+          extension ? DistributionFormat.new(extension, @response) : nil
+        end
+        # Get CSV dialect
+        @dialect = r['dialect']
+        # Extract schema
+        @schema = r['schema']
+        # Get path
+        @path = r['path']
+        @access_url = r['url']
+        # Set title
+        @title = @path || @uri
+      elsif r = options[:dcat_resource]
+        @title       = r[:title]
+        @description = r[:title]
+        @access_url  = r[:accessURL]
+      elsif r = options[:ckan_resource]
+        @title       = r[:title]
+        @description = r[:title]
+        @access_url  = r[:accessURL]
+        # Load HTTP Response for further use
+        if @access_url
+          @response = Curl::Easy.http_head(@access_url) do |c|
+                        c.follow_location = true
+                        c.useragent = "curb"
+                      end
+        end
+        @format = r[:format] ? DistributionFormat.new(r[:format], @response) : nil
+    end
+      # Set default CSV dialect
+      @dialect ||= {
+        "delimiter" => ","
+      }     
+    end
+
+    # A usable name for the distribution, unique within the {Dataset}.
+    #
+    # @return [String] a locally unique name
+    def title
+      @title
+    end
+    alias_method :name, :title
+
+    # An array of column headers for the distribution. Loaded from the schema, or from the file directly if no
+    # schema is present.
+    #
+    # @return [Array<String>] an array of column headers, as strings.
+    def headers
+      @headers ||= begin
+        if @schema
+          @schema['fields'].map{|x| x['id']}
+        else
+          data.headers
+        end
+      end
+    end
+    
+    # Whether the file that the distribution represents actually exists
+    #
+    # @return [Boolean] whether the HTTP response returns a success code or not
+    def exists?
+      if @access_url
+        @response.response_code != 404
+      end
+    end
+
+    # A CSV object representing the loaded data.
+    #
+    # @return [Array<Array<String>>] an array of arrays of strings, representing each row.
+    def data
+      @data ||= begin
+        if @path
+          datafile = @dataset.send(:load_file, @path)
+        elsif @access_url
+          datafile = RestClient.get @access_url rescue nil
+        end
+        if datafile
+          case format.extension
+          when :csv 
+            CSV.parse(
+              datafile, 
+              :headers => true,
+              :col_sep => @dialect["delimiter"]
+            )
+          else
+            nil
+          end
+        else
+          nil
+        end
+      rescue
+        nil
+      end
+    end
+
+  end  
+
+end

data/lib/data_kitten/distribution_format.rb

@@ -0,0 +1,73 @@
+module DataKitten
+
+  # A file format for a distribution
+  # 
+  # For instance CSV, XML, etc.
+  #
+  class DistributionFormat
+   
+    #@!attribute extension
+    #@return [Symbol] a symbol for the file extension. For instance, :csv.
+    attr_reader :extension
+
+    # Create a new DistributionFormat object with the relevant extension
+    #
+    # @param extension [String] the file extension for the format
+    def initialize(extension, response)
+      # Store extension as a lowercase symbol
+      @extension = extension.to_s.downcase.to_sym
+      # Store response for later use
+      @response = response
+      # Set up format lists
+      @@formats ||= {
+        csv:     { structured:  true, open:  true },
+        xls:     { structured:  true, open: false },
+        rdf:     { structured:  true, open:  true },
+        xml:     { structured:  true, open:  true },
+        wms:     { structured:  true, open:  true },
+        ods:     { structured:  true, open:  true },
+        rdfa:    { structured:  true, open:  true },
+        kml:     { structured:  true, open:  true },
+        rss:     { structured:  true, open:  true },
+        json:    { structured:  true, open:  true },
+        ical:    { structured:  true, open:  true },
+        sparql:  { structured:  true, open:  true },
+        kml:     { structured:  true, open:  true },
+        georss:  { structured:  true, open:  true },
+        shp:     { structured:  true, open:  true },
+        html:    { structured: false, open:  true },
+        doc:     { structured: false, open:  false },
+        pdf:     { structured: false, open:  true },  
+      }
+    end
+
+    # Is this a structured format?
+    #
+    # @return [Boolean] whether the format is machine-readable or not.
+    def structured?
+      @@formats[@extension][:structured] rescue nil
+    end
+
+    # Is this an open format?
+    #
+    # @return [Boolean] whether the format is open or not
+    def open?
+      @@formats[@extension][:open] rescue nil
+    end
+    
+    # Whether the format of the file matches the extension given by the data
+    #
+    # @return [Boolean] whether the MIME type given in the HTTP response matches the data or not
+    def matches?
+      begin
+        mimes = []
+        MIME::Types.type_for(@extension.to_s).each { |i| mimes << i.content_type }
+        !!(@response.content_type =~ /#{mimes.join('|')}/) || false
+      rescue
+        nil
+      end
+    end
+
+  end  
+
+end

data/lib/data_kitten/hosts.rb

@@ -0,0 +1,23 @@
+require 'data_kitten/hosts/github'
+require 'data_kitten/hosts/bitbucket'
+require 'data_kitten/hosts/gist'
+
+module DataKitten
+  
+  module Hosts
+
+    private
+
+    def detect_host
+      [
+        DataKitten::Hosts::Github,
+        DataKitten::Hosts::Bitbucket,
+        DataKitten::Hosts::Gist
+      ].each do |host|
+        extend host if host.supported?(@access_url)
+      end
+    end
+
+  end
+  
+end

data/lib/data_kitten/hosts/bitbucket.rb

@@ -0,0 +1,54 @@
+module DataKitten
+
+  module Hosts
+    
+    # Bitbucket host module. Automatically mixed into {Dataset} for datasets that are loaded from Bitbucket.
+    #
+    # @see Dataset
+    #
+    module Bitbucket
+
+      private
+
+      def self.supported?(uri)
+        uri =~ /\A(git|https?):\/\/[^\/]*bitbucket\.org\//
+      end
+      
+      public
+
+      # Where the dataset is hosted.
+      # @return [Symbol] +:bitbucket+
+      # @see Dataset#host
+      def host
+        :bitbucket
+      end
+
+      # Helper for generating Bitbucket URLs
+      #
+      # @param path [String] The path to append to the Bitbucket base URL.
+      #
+      # @return [String] The supplied path with the Bitbucket base URL prepended
+      #
+      # @example
+      #   dataset = Dataset.new(access_url: 'https://bitbucket.org/floppy/hot-drinks.git')
+      #   dataset.bitbucket_path           # => 'https://bitbucket.org/floppy/hot-drinks/'
+      #   dataset.bitbucket_path('pull-requests') # => 'https://bitbucket.org/floppy/hot-drinks/pull-requests'
+      def bitbucket_path(path = '')
+        "https://bitbucket.org/#{bitbucket_user_name}/#{bitbucket_repository_name}/#{path}"
+      end
+
+      private
+
+      def bitbucket_user_name
+        @bitbucket_user_name ||= uri.split('/')[-2]
+      end
+  
+      def bitbucket_repository_name
+        @bitbucket_repository_name ||= uri.split('/')[-1].split('.')[0]
+      end
+
+    end
+
+  end
+  
+end

data/lib/data_kitten/hosts/gist.rb

@@ -0,0 +1,50 @@
+module DataKitten
+
+  module Hosts
+    
+    # Gist host module. Automatically mixed into {Dataset} for datasets that are loaded from Gist.
+    #
+    # @see Dataset
+    #
+    module Gist
+
+      private
+
+      def self.supported?(uri)
+        uri =~ /\A(git|https?):\/\/gist\.github\.com\//
+      end
+      
+      public
+
+      # Where the dataset is hosted.
+      # @return [Symbol] +:gist+
+      # @see Dataset#host
+      def host
+        :gist
+      end
+
+      # Helper for generating Gist URLs
+      #
+      # @param path [String] The path to append to the Gist base URL.
+      #
+      # @return [String] The supplied path with the Gist base URL prepended
+      #
+      # @example
+      #   dataset = Dataset.new(access_url: 'git://gist.github.com/5633865.git')
+      #   dataset.gist_path           # => 'https://gist.github.com/5633865'
+      #   dataset.gist_path('download') # => 'https://gist.github.com/5633865/download'
+      def gist_path(path = '')
+        "https://gist.github.com/#{gist_repository_name}/#{path}"
+      end
+
+      private
+
+      def gist_repository_name
+        @gist_repository_name ||= uri.split('/')[-1].split('.')[0]
+      end
+
+    end
+
+  end
+  
+end

data/lib/data_kitten/hosts/github.rb

@@ -0,0 +1,54 @@
+module DataKitten
+
+  module Hosts
+    
+    # GitHub host module. Automatically mixed into {Dataset} for datasets that are loaded from GitHub.
+    #
+    # @see Dataset
+    #
+    module Github
+
+      private
+
+      def self.supported?(uri)
+        uri =~ /\A(git|https?):\/\/github\.com\//
+      end
+      
+      public
+
+      # Where the dataset is hosted.
+      # @return [Symbol] +:github+
+      # @see Dataset#host
+      def host
+        :github
+      end
+
+      # Helper for generating GitHub URLs
+      #
+      # @param path [String] The path to append to the GitHub base URL.
+      #
+      # @return [String] The supplied path with the GitHub base URL prepended
+      #
+      # @example
+      #   dataset = Dataset.new(access_url: 'git://github.com/theodi/dataset-metadata-survey.git')
+      #   dataset.github_path           # => 'https://github.com/theodi/dataset-metadata-survey/'
+      #   dataset.github_path('issues') # => 'https://github.com/theodi/dataset-metadata-survey/issues'
+      def github_path(path = '')
+        "https://github.com/#{github_user_name}/#{github_repository_name}/#{path}"
+      end
+
+      private
+
+      def github_user_name
+        @github_user_name ||= uri.split('/')[-2]
+      end
+  
+      def github_repository_name
+        @github_repository_name ||= uri.split('/')[-1].split('.')[0]
+      end
+
+    end
+
+  end
+  
+end

data/lib/data_kitten/license.rb

@@ -0,0 +1,39 @@
+module DataKitten
+
+  # A license for a {Dataset} or {Distribution}
+  #
+  class License
+
+    # @!attribute is
+    #   @return [String] a short ID that identifies the license.
+    attr_accessor :id
+    
+    # @!attribute name
+    #   @return [String] the human name of the license.
+    attr_accessor :name
+    
+    # @!attribute uri
+    #   @return [String] the URI for the license text.
+    attr_accessor :uri
+
+    # @!attribute type
+    #   @return [String] the type of information this license applies to. Could be +:data+ or +:content+.
+    attr_accessor :type
+
+    # Create a new License object.
+    #
+    # @param options [Hash] A set of options with which to initialise the license.
+    # @option options [String] :id the short ID for the license
+    # @option options [String] :name the human name for the license
+    # @option options [String] :uri the URI of the license text
+    # @option options [String] :type the type of information covered by this license.
+    def initialize(options)
+      @id = options[:id]
+      @name = options[:name]
+      @uri = options[:uri]
+      @type = options[:type]
+    end
+
+  end  
+
+end