data_kitten 1.0.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MGU0NTIxNWM3YjhlNDU1MDNiYTk2YzkwYzE3ZjZmODU5YWEyMDE4NQ==
+  data.tar.gz: !binary |-
+    YWNhOTQ2ODVjZTI0MTUxYmI3MjRhNjZlODBkNjQ5NmZjNjc5MTI1NA==
+SHA512:
+  metadata.gz: !binary |-
+    Zjc4OWRjYmRmYzcxNzJjZmNkMGVmMmRhZmIyODVmMTAwZmJmZGZlNzdlZmNl
+    Yjk5M2ZlYzU4NTllMTcyYzdkNTk4NTJkMGQ2NzcxMDhhY2MzNjZiOTcwZWU1
+    M2NhOWYwY2Y5M2NhZWRhZjUyM2M5ODg2YjM5NDVkYTlhZDgyMzM=
+  data.tar.gz: !binary |-
+    OWUyMjlhYzA1YzA0NDY4ZGE1OTNjMTA3ZDA2MjMyNDkzYmVlZDY3N2ZlYzIw
+    MWNmOTNmZjZkNzM2YzE0OTYxZjRkNWMzOGU5NmMwMWRjMTM0NTFkYTZiYmY4
+    YzFjMzljNzcyYzU3ZTQ3YmQzZDJjNWFiOWVhZmNlYjU3NGNmODA=

data/LICENSE.md

@@ -0,0 +1,20 @@
+Copyright 2013 The Open Data Institute
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,73 @@
+[![Build Status](http://img.shields.io/travis/theodi/data_kitten.svg)](https://travis-ci.org/theodi/data_kitten)
+[![Dependency Status](http://img.shields.io/gemnasium/theodi/data_kitten.svg)](https://gemnasium.com/theodi/data_kitten)
+[![Coverage Status](http://img.shields.io/coveralls/theodi/data_kitten.svg)](https://coveralls.io/r/theodi/data_kitten)
+[![Code Climate](http://img.shields.io/codeclimate/github/theodi/data_kitten.svg)](https://codeclimate.com/github/theodi/data_kitten)
+[![Gem Version](http://img.shields.io/gem/v/data_kitten.svg)](https://rubygems.org/gems/data_kitten)
+[![License](http://img.shields.io/:license-mit-blue.svg)](http://theodi.mit-license.org)
+[![Badges](http://img.shields.io/:badges-7/7-ff6799.svg)](https://github.com/pikesley/badger)
+
+# data_kitten
+
+![DATAS - I HAZ THEM](https://gs1.wac.edgecastcdn.net/8019B6/data.tumblr.com/67399f2b335ef62d562dc9eb41c0db16/tumblr_mmy9g7rA8M1s4aj1ho1_500.jpg)
+
+A collection of classes that represent Datasets and other concepts, modeled on [DCAT](http://www.w3.org/TR/vocab-dcat/)
+
+The module is designed to automatically interrogate data sources and give back data 
+and metadata in a consistent format. The best starting place is probably by having a look at `Dataset`.
+
+It is designed to handle data from multiple `Sources` (such as git repositories, local files, remote URLs), 
+`Hosts` (GitHub, etc), and `PublishingFormats` (DataPackage, RDFa, microdata, DSPL, etc).
+
+Currently supports Datapackages in git repositories (including but not limited to GitHub repos). 
+Wider support will follow.
+
+# Documentation
+
+Full YARD documentation is available on [Rubydoc.info](http://rubydoc.info/github/theodi/data_kitten/master/frames).
+
+# Licence
+
+This code is open source under the MIT license. See the LICENSE.md file for full details.
+
+# Requirements
+
+* Git ~> 1.2.6
+
+# Usage
+
+Pop the gem into your Gemfile:
+
+        gem 'data_kitten', :git => "git://github.com/theodi/data_kitten.git"
+
+Require if you need to:
+
+	require 'data_kitten'
+	
+Request a dataset:
+	
+	dataset = DataKitten::Dataset.new(access_url: "https://github.com/theodi/dataset-mod-disposals.git")
+	
+Use the results:
+
+	dataset.supported?
+	dataset.origin
+	dataset.host
+	dataset.data_title
+	dataset.documentation_url
+	dataset.release_type
+	dataset.time_sensitive?
+	dataset.publishing_format
+	dataset.maintainers
+	dataset.publishers
+	dataset.licenses
+	dataset.contributors
+	dataset.crowdsourced?
+	dataset.contributor_agreement_url
+	dataset.distributions
+	dataset.change_history
+	
+	# And more to come!
+
+See example usage in a Rails project at [https://github.com/theodi/git-data-viewer](https://github.com/theodi/git-data-viewer)
+
+![actual_data_kitten](http://i.imgur.com/wXZEkh7.gif)

data/bin/data_kitten

@@ -0,0 +1,22 @@
+#!/usr/bin/env ruby
+$:.unshift File.join( File.dirname(__FILE__), "..", "lib")
+
+require 'data_kitten'
+require 'pp'
+
+if ARGV.length == 0
+  puts "Usage: data_kitten <access_url>"
+  exit 1
+end
+
+dataset = DataKitten::Dataset.new(access_url: ARGV[0])
+
+if dataset.publishing_format == nil
+  puts "Unable to determine format for dataset metadata"
+  exit 1
+end
+
+(dataset.public_methods - Object.public_methods).sort.delete_if {|x| x.to_s =~ /=/ }.each do |method|
+    puts "#{method}: #{dataset.send(method).pretty_inspect}"
+end
+

data/lib/data_kitten.rb

@@ -0,0 +1,43 @@
+require 'csv'
+require 'uri'
+require 'cgi'
+require 'git'
+require 'json'
+require 'rest-client'
+require 'rdf'
+require 'linkeddata'
+require 'nokogiri'
+require 'uri'
+require 'curb'
+require 'datapackage'
+
+require 'data_kitten/license'
+require 'data_kitten/rights'
+require 'data_kitten/agent'
+require 'data_kitten/source'
+require 'data_kitten/temporal'
+require 'data_kitten/dataset'
+require 'data_kitten/distribution_format'
+require 'data_kitten/distribution'
+
+# A collection of classes that represent Datasets and other concepts, modeled on {http://www.w3.org/TR/vocab-dcat/ DCAT}.
+#
+# The module is designed to automatically interrogate data sources and give back data and metadata in a consistent
+# format. The best starting place is probably by having a look at {Dataset}.
+#
+# It is designed to handle data from multiple {Sources} (such as git repositories, local files, remote URLs), 
+# {Hosts} (GitHub, etc), and {PublishingFormats} (DataPackage, RDFa, microdata, DSPL, etc).
+#
+# Currently supports Datapackages in git repositories (including but not limited to GitHub repos). Wider support will follow.
+#
+# https://gs1.wac.edgecastcdn.net/8019B6/data.tumblr.com/67399f2b335ef62d562dc9eb41c0db16/tumblr_mmy9g7rA8M1s4aj1ho1_500.jpg
+#
+# @example Load a Dataset from a git repository
+#   dataset = Dataset.new(access_url: 'git://github.com/theodi/dataset-metadata-survey.git')
+#   dataset.supported?                # => true
+#   dataset.origin                    # => :git
+#   dataset.host                      # => :github
+#   dataset.publishing_format         # => :datapackage
+#   dataset.distributions             # => [Distribution<#1>, Distribution<#2>]
+#   dataset.distributions[0].headers  # => ['col1', 'col2']
+#   dataset.distributions[0].data[0]  # => {'col1' => 'value_1', 'col2' => 'value_2'}

data/lib/data_kitten/agent.rb

@@ -0,0 +1,38 @@
+module DataKitten
+
+  # A person or organisation.
+  #
+  # Naming is based on {http://xmlns.com/foaf/spec/#term_Agent foaf:Agent}, but with useful aliases for other vocabularies.
+  class Agent
+
+    # Create a new Agent
+    # 
+    # @param [Hash] options the details of the Agent.
+    # @option options [String] :name The Agent's name
+    # @option options [String] :homepage The homepage URL for the Agent
+    # @option options [String] :mbox Email address for the Agent
+    #
+    def initialize(options)
+      @name = options[:name]
+      @homepage = options[:homepage]
+      @mbox = options[:mbox]
+    end
+
+    # @!attribute name
+    #   @return [String] the name of the Agent
+    attr_accessor :name
+    
+    # @!attribute homepage
+    #   @return [String] the homepage URL of the Agent
+    attr_accessor :homepage
+    alias_method :url, :homepage
+    alias_method :uri, :homepage
+
+    # @!attribute mbox
+    #   @return [String] the email address of the Agent
+    attr_accessor :mbox
+    alias_method :email, :mbox
+
+  end  
+
+end

data/lib/data_kitten/dataset.rb

@@ -0,0 +1,227 @@
+require 'data_kitten/origins'
+require 'data_kitten/hosts'
+require 'data_kitten/publishing_formats'
+
+module DataKitten
+
+  # Represents a single dataset from some origin (see {http://www.w3.org/TR/vocab-dcat/#class-dataset dcat:Dataset} 
+  # for relevant vocabulary).
+  #
+  # Designed to be created with a URI to the dataset, and then to work out metadata from there.
+  # 
+  # Currently supports Datasets hosted in Git (and optionally on GitHub), and which 
+  # use the Datapackage metadata format.
+  #
+  # @example Load a Dataset from a git repository
+  #   dataset = Dataset.new(access_url: 'git://github.com/theodi/dataset-metadata-survey.git')
+  #   dataset.supported?         # => true
+  #   dataset.origin             # => :git
+  #   dataset.host               # => :github
+  #   dataset.publishing_format  # => :datapackage
+  #   
+  class Dataset
+
+    include DataKitten::Origins
+    include DataKitten::Hosts
+    include DataKitten::PublishingFormats
+  
+    # @!attribute access_url
+    #   @return [String] the URL that gives access to the dataset
+    attr_accessor :access_url
+    alias_method :uri, :access_url
+    alias_method :url, :access_url
+
+    # Create a new Dataset object
+    # 
+    # @param [Hash] options the details of the Dataset.
+    # @option options [String] :access_url A URL that can be used to access the Dataset. 
+    #                                      The class will attempt to auto-load metadata from this URL.
+    #
+    def initialize(options)
+      @access_url = options[:access_url]
+      detect_origin
+      detect_host
+      detect_publishing_format
+    end
+  
+    # Can metadata be loaded for this Dataset?
+    #
+    # @return [Boolean] true if metadata can be loaded, false if it's 
+    #                   an unknown origin type, or has an unknown metadata format.
+    def supported?
+      !(origin.nil? || publishing_format.nil?)
+    end
+  
+    # The origin type of the dataset.
+    #
+    # @return [Symbol] The origin type. For instance, datasets loaded from git 
+    #                  repositories will return +:git+. If no origin type is 
+    #                  identified, will return +nil+.
+    def origin
+      nil
+    end
+    
+    # Where the dataset is hosted.
+    #
+    # @return [Symbol] The host. For instance, data loaded from github repositories 
+    #                  will return +:github+. This can be used to control extra host-specific
+    #                  behaviour if required. If no host type is identified, will return +nil+.
+    def host
+      nil
+    end
+    
+    # The human-readable title of the dataset.
+    #
+    # @return [String] the title of the dataset.
+    def data_title
+      nil
+    end
+    
+    # A brief description of the dataset
+    #
+    # @return [String] the description of the dataset.
+    def description
+      nil
+    end
+    
+    # Keywords for the dataset
+    #
+    # @return [Array<string>] an array of keywords
+    def keywords
+      []
+    end
+  
+    # Human-readable documentation for the dataset.
+    #
+    # @return [String] the URL of the documentation.
+    def documentation_url
+      nil
+    end
+  
+    # What type of dataset is this?
+    # Options are: +:web_service+ for API-accessible data, or +:one_off+ for downloadable data dumps.
+    #
+    # @return [Symbol] the release type.
+    def release_type
+      false
+    end
+    
+    # Date the dataset was released
+    #
+    # @return [Date] the release date of the dataset
+    def issued
+      nil
+    end
+    alias_method :release_date, :issued
+    
+    # Date the dataset was last modified
+    #
+    # @return [Date] the dataset's last modified date
+    def modified
+      nil
+    end
+    
+    # The temporal coverage of the dataset
+    #
+    # @return [Object<Temporal>] the start and end dates of the dataset's temporal coverage
+    def temporal
+      nil
+    end
+    
+    # Where the data is sourced from
+    #
+    # @return [Array<Source>] the sources of the data, each as a Source object.
+    def sources
+      []
+    end
+  
+    # Is the information time-sensitive?
+    #
+    # @return [Boolean] whether the information will go out of date.
+    def time_sensitive?
+      false
+    end
+    
+    # The publishing format for the dataset.
+    #
+    # @return [Symbol] The format. For instance, datasets that publish metadata in
+    #                  Datapackage format will return +:datapackage+. If no format 
+    #                  is identified, will return +nil+.
+    def publishing_format
+      nil
+    end
+
+    # A list of maintainers
+    #
+    # @return [Array<Agent>] An array of maintainers, each as an Agent object.
+    def maintainers
+      []
+    end
+
+    # A list of publishers
+    #
+    # @return [Array<Agent>] An array of publishers, each as an Agent object.
+    def publishers
+      []
+    end
+
+    # A list of licenses
+    #
+    # @return [Array<License>] An array of licenses, each as a License object.
+    def licenses
+      []
+    end
+    
+    # The rights statment for the data
+    #
+    # @return [Object<Rights>] How the content and data can be used, as well as copyright notice and attribution URL
+    def rights
+      nil
+    end
+
+    # A list of contributors
+    #
+    # @return [Array<Agent>] An array of contributors to the dataset, each as an Agent object.
+    def contributors
+      []
+    end
+    
+    # Has the data been crowdsourced?
+    #
+    # @return [Boolean] Whether the data has been crowdsourced or not.
+    def crowdsourced?
+      false
+    end
+
+    # The URL of the contributor license agreement
+    #
+    # @return [String] A URL for the agreement that contributors accept.
+    def contributor_agreement_url
+      nil
+    end
+
+    # A list of distributions. Has aliases for popular alternative vocabularies.
+    #
+    # @return [Array<Distribution>] An array of Distribution objects.
+    def distributions
+      []
+    end
+    alias_method :files, :distributions
+    alias_method :resources, :distributions
+    
+    # How frequently the data is updated.
+    #
+    # @return [String] The frequency of update expressed as a dct:Frequency.
+    def update_frequency
+      nil
+    end
+
+    # A history of changes to the Dataset
+    #
+    # @return [Array] An array of changes. Exact format depends on the origin and publishing format.
+    def change_history
+      []
+    end
+
+  end
+end