data_kitten 1.0.0

data/lib/data_kitten/origins.rb

@@ -0,0 +1,28 @@
+require 'data_kitten/origins/git'
+require 'data_kitten/origins/web_service'
+require 'data_kitten/origins/html'
+require 'data_kitten/origins/linked_data'
+
+module DataKitten
+  
+  module Origins
+
+    private
+
+    def detect_origin
+      [
+        DataKitten::Origins::Git,
+        DataKitten::Origins::HTML,
+        DataKitten::Origins::WebService,
+        DataKitten::Origins::LinkedData,
+      ].each do |origin|
+        if origin.supported?(@access_url)
+          extend origin 
+          break
+        end
+      end
+    end
+
+  end
+  
+end

data/lib/data_kitten/origins/git.rb

@@ -0,0 +1,66 @@
+module DataKitten
+  
+  module Origins
+    
+    # Git origin module. Automatically mixed into {Dataset} for datasets that are loaded from Git repositories.
+    #
+    # @see Dataset
+    #
+    module Git
+  
+      private
+  
+      def self.supported?(uri)
+        uri =~ /\A(git|https?):\/\/.*\.git\Z/
+      end
+
+      public
+
+      # The origin type of the dataset.
+      # @return [Symbol] +:git+
+      # @see Dataset#origin
+      def origin
+        :git
+      end
+
+      # A history of changes to the Dataset, taken from the full git changelog
+      # @see Dataset#change_history
+      def change_history
+        @change_history ||= begin
+          repository.log.map{|commit| commit}
+        end
+      end
+
+      protected
+  
+      def load_file(path)
+        # Make sure we have a working copy
+        repository
+        # read file 
+        File.read(File.join(working_copy_path, path))
+      end
+  
+      private
+  
+      def working_copy_path
+        # Create holding directory
+        FileUtils.mkdir_p(File.join(File.dirname(__FILE__), '..', '..', '..', 'tmp', 'repositories'))
+        # generate working copy dir
+        File.join(File.dirname(__FILE__), '..', '..', '..', 'tmp', 'repositories', @access_url.gsub('/','-'))
+      end
+
+      def repository
+        @repository ||= begin
+          repo = ::Git.open(working_copy_path)
+          repo.pull("origin", "master")
+          repo
+        rescue ArgumentError
+          repo = ::Git.clone(@access_url, working_copy_path)
+        end
+      end  
+  
+    end
+
+  end
+
+end

data/lib/data_kitten/origins/html.rb

@@ -0,0 +1,32 @@
+module DataKitten
+  
+  module Origins
+    
+    # HTML origin module. Automatically mixed into {Dataset} for datasets that are accessed through an API.
+    #
+    # @see Dataset
+    #
+    module HTML
+  
+      private
+  
+      def self.supported?(uri)
+        RestClient.get(uri).headers[:content_type] =~ /text\/html/
+      rescue
+          false
+      end
+
+      public
+
+      # The origin type of the dataset.
+      # @return [Symbol] +:html+
+      # @see Dataset#origin
+      def origin
+        :html
+      end
+
+    end
+
+  end
+
+end

data/lib/data_kitten/origins/linked_data.rb

@@ -0,0 +1,37 @@
+module DataKitten
+  
+  module Origins
+    
+    # Linked Data origin module. Automatically mixed into {Dataset} for datasets that are accessed through an API.
+    #
+    # @see Dataset
+    #
+    module LinkedData
+  
+      private
+  
+      def self.supported?(uri)
+        content_type = RestClient.head(uri).headers[:content_type]
+        return nil unless content_type
+        
+        return RDF::Format.content_types.keys.include?( 
+            content_type.split(";").first )    
+            
+      rescue
+          false
+      end
+
+      public
+
+      # The origin type of the dataset.
+      # @return [Symbol] +:linkeddata+
+      # @see Dataset#origin
+      def origin
+        :linkeddata
+      end
+
+    end
+
+  end
+
+end

data/lib/data_kitten/origins/web_service.rb

@@ -0,0 +1,30 @@
+module DataKitten
+  
+  module Origins
+    
+    # Web service origin module. Automatically mixed into {Dataset} for datasets that are accessed through an API.
+    #
+    # @see Dataset
+    #
+    module WebService
+  
+      private
+  
+      def self.supported?(uri)
+        false
+      end
+
+      public
+
+      # The origin type of the dataset.
+      # @return [Symbol] +:web_service+
+      # @see Dataset#origin
+      def origin
+        :web_service
+      end
+
+    end
+
+  end
+
+end

data/lib/data_kitten/publishing_formats.rb

@@ -0,0 +1,28 @@
+require 'data_kitten/publishing_formats/datapackage'
+require 'data_kitten/publishing_formats/rdfa'
+require 'data_kitten/publishing_formats/linked_data'
+require 'data_kitten/publishing_formats/ckan'
+
+module DataKitten
+  
+  module PublishingFormats
+
+    private
+
+    def detect_publishing_format
+      [
+        DataKitten::PublishingFormats::Datapackage,
+        DataKitten::PublishingFormats::CKAN,
+        DataKitten::PublishingFormats::RDFa,
+        DataKitten::PublishingFormats::LinkedData
+      ].each do |format|
+        if format.supported?(self)
+          extend format 
+          break
+        end
+      end
+    end
+
+  end
+  
+end

data/lib/data_kitten/publishing_formats/ckan.rb

@@ -0,0 +1,187 @@
+module DataKitten
+
+  module PublishingFormats
+
+    module CKAN
+
+      @@metadata = nil
+
+      private
+
+      def self.supported?(instance)
+        uri = URI(instance.uri)
+        package = uri.path.split("/").last
+        # If the package is a UUID - it's more than likely to be a CKAN ID
+        if package.match(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/)
+          @@id = package
+        else
+
+          results = RestClient.get "#{uri.scheme}://#{uri.host}/api/3/action/package_show", {:params => {:id => package}} rescue ""
+
+          if results == ""
+            results = RestClient.get "#{uri.scheme}://#{uri.host}/api/2/rest/dataset/#{package}"
+          end
+
+          result = JSON.parse results
+          @@id = result["result"]["id"] rescue result["id"]
+          @@metadata = JSON.parse RestClient.get "#{uri.scheme}://#{uri.host}/api/rest/package/#{@@id}"
+        end
+      rescue
+        false
+      end
+
+      public
+
+      # The publishing format for the dataset.
+      # @return [Symbol] +:ckan+
+      # @see Dataset#publishing_format
+      def publishing_format
+        :ckan
+      end
+
+      # The human-readable title of the dataset.
+      #
+      # @see Dataset#data_title
+      def data_title
+        metadata["title"] rescue nil
+      end
+
+      # A brief description of the dataset
+      #
+      # @see Dataset#description
+      def description
+        metadata["notes"] rescue nil
+      end
+
+      # Keywords for the dataset
+      #
+      # @see Dataset#keywords
+      def keywords
+        keywords = []
+        metadata["tags"].each do |tag|
+          keywords << tag
+        end
+        return keywords
+      rescue
+        []
+      end
+
+      # A list of publishers.
+      #
+      # @see Dataset#publishers
+      def publishers
+        id = metadata['organization']['id'] || metadata['groups'][0]
+        fetch_publisher(id)
+      rescue
+        []
+      end
+
+      # A list of licenses.
+      #
+      # @see Dataset#licenses
+      def licenses
+        uri = metadata["license_url"] || metadata["extras"]["licence_url"] rescue nil
+        name = metadata["license_title"] || metadata["extras"]["licence_url_title"] rescue nil
+        [
+          License.new(:id => metadata["license_id"],
+                      :uri => uri,
+                      :name => name
+                      )
+        ]
+      rescue
+        []
+      end
+
+      # A list of distributions, referred to as +resources+ by Datapackage.
+      #
+      # @see Dataset#distributions
+      def distributions
+        distributions = []
+        metadata["resources"].each do |resource|
+          distribution = {
+            :title => resource["description"],
+            :accessURL => resource["url"],
+            :format => resource["format"]
+          }
+          distributions << Distribution.new(self, ckan_resource: distribution)
+        end
+        return distributions
+      rescue
+        nil
+      end
+
+      # How frequently the data is updated.
+      #
+      # @see Dataset#update_frequency
+      def update_frequency
+        metadata["extras"]["update_frequency"] || metadata["extras"]["frequency-of-update"] rescue nil
+      end
+
+      # Date the dataset was released
+      #
+      # @see Dataset#issued
+      def issued
+        Date.parse metadata["metadata_created"] rescue nil
+      end
+
+      # Date the dataset was modified
+      #
+      # @see Dataset#modified
+      def modified
+        Date.parse metadata["metadata_modified"] rescue nil
+      end
+
+      # The temporal coverage of the dataset
+      #
+      # @see Dataset#temporal
+      def temporal
+        start_date = Date.parse metadata["extras"]["temporal_coverage-from"] rescue nil
+        end_date = Date.parse metadata["extras"]["temporal_coverage-to"] rescue nil
+        Temporal.new(:start => start_date, :end => end_date)
+      end
+
+      private
+
+      def metadata
+        @@metadata
+      end
+
+      def select_extras(group, key)
+        extra = group["extras"][key] rescue ""
+        if extra == ""
+          extra = group['result']['extras'].select {|e| e["key"] == key }.first['value'] rescue ""
+        end
+        extra
+      end
+
+      def fetch_publisher(id)
+        uri = parsed_uri
+        [
+          "#{uri.scheme}://#{uri.host}/api/rest/group/#{id}",
+          "#{uri.scheme}://#{uri.host}/api/3/action/group_show?id=#{id}",
+          "#{uri.scheme}://#{uri.host}/api/3/action/organization_show?id=#{id}"
+        ].each do |uri|
+          begin
+            @group = JSON.parse RestClient.get uri
+            break
+          rescue RestClient::ResourceNotFound
+            nil
+          end
+        end
+
+        [
+          Agent.new(
+                    :name => @group["display_name"] || @group["result"]["name"],
+                    :homepage => select_extras(@group, "website-url"),
+                    :mbox => select_extras(@group, "contact-email")
+                    )
+        ]
+      end
+
+      def parsed_uri
+        URI(self.uri)
+      end
+
+    end
+  end
+end

data/lib/data_kitten/publishing_formats/datapackage.rb

@@ -0,0 +1,169 @@
+module DataKitten
+
+  module PublishingFormats
+    
+    # Datapackage metadata format module. Automatically mixed into {Dataset} for datasets that include a +datapackage.json+.
+    #
+    # @see Dataset
+    #
+    module Datapackage
+
+      private
+
+      def self.supported?(instance)
+        begin
+          if instance.send(:origin) == :git
+            metadata = instance.send(:load_file, "datapackage.json")
+            datapackage = DataPackage::Package.new( JSON.parse( metadata ) )
+            return datapackage.datapackage_version != nil
+          else
+            datapackage = DataPackage::Package.new( instance.uri )
+            return datapackage.datapackage_version != nil
+          end
+        rescue => e
+          false
+        end
+      end
+
+      public
+
+      # The publishing format for the dataset.
+      # @return [Symbol] +:datapackage+
+      # @see Dataset#publishing_format
+      def publishing_format
+        :datapackage
+      end
+      
+      # A list of maintainers.
+      #
+      # @see Dataset#maintainers
+      def maintainers
+        package.maintainers.map do |x|
+          Agent.new(:name => x['name'], :uri => x['web'], :email => x['email'])
+        end
+      end
+
+      # A list of publishers.
+      #
+      # @see Dataset#publishers
+      def publishers
+        package.publisher.map do |x|
+          Agent.new(:name => x['name'], :uri => x['web'], :email => x['email'])
+        end
+      end
+
+      # A list of licenses.
+      #
+      # @see Dataset#licenses
+      def licenses
+        package.licenses.map do |x| 
+          License.new(:id => x['id'], :uri => x['url'], :name => x['name'])
+        end
+      end
+
+      def rights
+         if package.property("rights")
+            Rights.new( ( package.property("rights", [])).each_with_object({}){|(k,v), h| h[k.to_sym] = v} )
+         else
+            nil
+         end 
+      end
+      
+      # A list of contributors.
+      #
+      # @see Dataset#contributors
+      def contributors
+        package.contributors.map do |x|
+          Agent.new(:name => x['name'], :uri => x['web'], :email => x['email'])
+        end
+      end
+
+      # A list of distributions, referred to as +resources+ by Datapackage.
+      #
+      # @see Dataset#distributions
+      def distributions
+        package.resources.map { |resource| Distribution.new(self, datapackage_resource: resource) }
+      end
+  
+      # The human-readable title of the dataset.
+      #
+      # @see Dataset#data_title
+      def data_title
+        package.title || package.name
+      end
+      
+      # A brief description of the dataset
+      #
+      # @see Dataset#description
+      def description
+        package.description
+      end
+      
+      # Keywords for the dataset
+      #
+      # @see Dataset#keywords
+      def keywords
+        package.keywords
+      end
+      
+      # Where the data is sourced from
+      #
+      # @see Dataset#sources
+      def sources
+        package.sources.map do |x| 
+          Source.new(:label => x['name'], :resource => x['web'])
+        end
+      end
+      
+      # Date the dataset was modified
+      def modified
+        package.last_modified
+      end
+            
+      # A history of changes to the Dataset.
+      # 
+      # If {Dataset#source} is +:git+, this is the git changelog for the actual distribution files, rather
+      # then the full unfiltered log.
+      #
+      # @return [Array] An array of changes. Exact format depends on the source.
+      # 
+      # @see Dataset#change_history
+      def change_history
+        @change_history ||= begin
+          if origin == :git
+            # Get a log for each file in the local repo
+            logs = distributions.map do |file|
+              if file.path
+                log = repository.log.path(file.path)
+                # Convert to list of commits
+                log.map{|commit| commit}
+              else
+                []
+              end
+            end
+            # combine all logs, make unique, and re-sort in date order
+            logs.flatten.uniq.sort_by{|x| x.committer.date}.reverse
+          else
+            []
+          end
+        end
+      end
+
+      private
+      
+      def package
+        if !@datapackage
+          if origin == :git
+            metadata = load_file("datapackage.json")
+            @datapackage = DataPackage::Package.new( JSON.parse( metadata ) )
+          else
+            @datapackage = DataPackage::Package.new( access_url )
+          end        
+        end
+        @datapackage         
+      end
+    end
+
+  end
+  
+end