datapackage 0.0.4 → 0.1.0

data/lib/datapackage/registry.rb

@@ -0,0 +1,81 @@
+module DataPackage
+  ##
+  # Allow loading Data Package profiles from a registry.
+
+  class Registry
+
+    DEFAULT_REGISTRY_URL = 'http://schemas.datapackages.org/registry.csv'
+    DEFAULT_REGISTRY_PATH = File.join(File.expand_path(File.dirname(__FILE__)), '..', '..', 'datapackage', 'schemas', 'registry.csv')
+
+    attr_reader :base_path
+
+    def initialize(registry_path_or_url = DEFAULT_REGISTRY_PATH)
+      registry_path_or_url ||= DEFAULT_REGISTRY_PATH
+      if File.file?(registry_path_or_url)
+        @base_path = File.dirname(
+          File.absolute_path(registry_path_or_url)
+        )
+      end
+      @profiles = {}
+      @registry = get_registry(registry_path_or_url)
+    end
+
+    def get(profile_id)
+      @profiles[profile_id] ||= get_profile(profile_id)
+    end
+
+    def available_profiles
+      @registry
+    end
+
+    private
+
+      def get_profile(profile_id)
+        profile_metadata = @registry[profile_id]
+        return if profile_metadata.nil?
+
+        path = get_absolute_path(profile_metadata[:schema_path])
+
+        if path && File.file?(path)
+          load_json(path)
+        else
+          url = profile_metadata[:schema]
+          load_json(url)
+        end
+      end
+
+      def get_registry(registry_path_or_url)
+        begin
+          csv = parse_csv(registry_path_or_url)
+          registry = {}
+          csv.each { |row| registry[row.fetch(:id)] = Hash[row.headers.zip(row.fields)] }
+        rescue KeyError, OpenURI::HTTPError, Errno::ENOENT
+          raise(RegistryError)
+        end
+        registry
+      end
+
+      def parse_csv(path_or_url)
+        csv = open(path_or_url).read
+        if csv.match(/,/)
+          CSV.new(csv, headers: :first_row, header_converters: :symbol)
+        else
+          raise RegistryError
+        end
+      end
+
+      def get_absolute_path(relative_path)
+        File.join(@base_path, relative_path)
+      rescue TypeError
+        nil
+      end
+
+      def load_json(path)
+        json = open(path).read
+        JSON.parse(json)
+      rescue JSON::ParserError, OpenURI::HTTPError
+        raise RegistryError
+      end
+
+  end
+end

data/lib/datapackage/resource.rb

@@ -0,0 +1,79 @@
+module DataPackage
+  class Resource < Hash
+
+    def initialize(resource, base_path = '')
+      self.merge! resource
+    end
+
+    def self.load(resource, base_path = '')
+      # This returns if there are no alternative ways to access the data OR there
+      # is a base_path which is a URL
+      if is_url?(resource, base_path)
+        RemoteResource.new(resource, base_path)
+      else
+        # If there's a data attribute, we definitely want an inline resource
+        if resource['data']
+          InlineResource.new(resource)
+        else
+          # If the file exists - we want a local resource
+          if file_exists?(resource, base_path)
+            LocalResource.new(resource, base_path)
+          # If it doesn't exist and there's a URL to grab the data from, we want
+          # a remote resource
+          elsif resource['url']
+            RemoteResource.new(resource, base_path)
+          end
+        end
+      end
+    end
+
+    def self.file_exists?(resource, base_path)
+      path = resource['path']
+      path = File.join(base_path, path) if base_path != ''
+      File.exists?(path)
+    end
+
+    def self.is_url?(resource, base_path)
+      return true if resource['url'] != nil && resource['path'] == nil && resource['data'] == nil
+      return true if base_path.start_with?('http')
+    end
+
+  end
+
+  class LocalResource < Resource
+
+    def initialize(resource, base_path = '')
+      @base_path = base_path
+      @path = resource['path']
+      super
+    end
+
+    def data
+      @path = File.join(@base_path, @path) if @base_path != ''
+      open(@path).read
+    end
+
+  end
+
+  class InlineResource < Resource
+    def data
+      self['data']
+    end
+  end
+
+  class RemoteResource < Resource
+
+    def initialize(resource, base_url = '')
+      @base_url = base_url
+      @url = resource['url']
+      @path = resource['path']
+      super
+    end
+
+    def data
+      url = @url ? @url : URI.join(@base_url, @path)
+      open(url).read
+    end
+
+  end
+end

data/lib/datapackage/schema.rb

@@ -0,0 +1,111 @@
+module DataPackage
+  class Schema < Hash
+
+    attr_reader :schema
+
+    def initialize(schema, options = {})
+      @registry_url = options[:registry_url]
+      if schema.class == Hash
+        self.merge! schema
+      elsif schema.class == Symbol
+        self.merge! get_schema_from_registry schema
+      elsif schema.class == String
+        self.merge! load_schema(schema)
+      else
+        raise SchemaException.new "Schema must be a URL, path, Hash or registry-identifier"
+      end
+    end
+
+# https://gist.github.com/vdw/f3c832df8ce271a036f2
+    def hash_to_slashed_path(hash, path = "")
+      return {} unless hash
+      hash.each_with_object({}) do |(k, v), ret|
+        key = path + k.to_s
+
+        if v.is_a? Hash
+          ret.merge! hash_to_slashed_path(v, key.to_s + "/")
+        else
+          ret[key] = v
+        end
+      end
+    end
+
+    def dereference_schema path_or_url, schema
+      paths = hash_to_slashed_path schema
+      ref_keys = paths.keys.select { |p| p =~ /\$ref/ }
+      if ref_keys
+        ref_keys = [ref_keys] unless ref_keys.is_a? Array
+
+        ref_keys.each do |key|
+          path = key.split('/')[0..-2]
+
+          replacement = resolve(schema.dig(*path, '$ref'), path_or_url, schema)
+
+          s = "schema#{path.map { |k| "['#{k}']" }.join}.merge! replacement"
+          eval s
+          s = "schema#{path.map { |k| "['#{k}']" }.join}.delete '$ref'"
+          eval s
+        end
+      end
+
+      schema
+    end
+
+    def resolve reference, path_or_url, schema
+      base_path = base_path path_or_url
+      filename, reference = reference.split '#'
+      if filename == ''
+        schema['define'][reference.split('/').last]
+      else
+        dereference_schema("#{base_path}/#{filename}", get_definitions(filename, base_path)).dig(*reference.split('/').reject(&:empty?))
+      end
+    end
+
+    def get_definitions filename, base_path
+      JSON.parse open("#{base_path}/#{filename}").read
+    end
+
+    def base_path path_or_url
+      if path_or_url =~ /\A#{URI::regexp}\z/
+        uri = URI.parse path_or_url
+        return "#{uri.scheme}://#{uri.host}#{File.dirname uri.path}".chomp('/')
+      else
+
+        if File.directory?(path_or_url)
+          return path_or_url
+        else
+          return File.expand_path File.dirname path_or_url
+        end
+      end
+    end
+
+    def load_schema(path_or_url)
+      json = open(path_or_url).read
+      schema = JSON.parse json
+      s = dereference_schema path_or_url, schema
+
+    rescue OpenURI::HTTPError => e
+      raise SchemaException.new "Schema URL returned #{e.message}"
+
+    rescue JSON::ParserError
+      raise SchemaException.new 'Schema is not valid JSON'
+
+    rescue Errno::ENOENT
+      raise SchemaException.new "Path '#{path_or_url}' does not exist"
+    end
+
+    def get_schema_from_registry schema
+      d = DataPackage::Registry.new(@registry_url)
+      dereference_schema (@registry_url || d.base_path), d.get(schema.to_s)
+    end
+
+    def valid?(package)
+      JSON::Validator.validate(self, package)
+    end
+
+    def validation_errors(package)
+      JSON::Validator.fully_validate(self, package)
+    end
+
+  end
+end

data/lib/datapackage/version.rb

@@ -1,3 +1,3 @@
 module DataPackage
-  VERSION = "0.0.4"
+  VERSION = "0.1.0"
 end

data/lib/datapackage.rb

@@ -5,7 +5,12 @@ require 'csv'
 require 'json'
 require 'json-schema'
 require 'rest-client'
+require 'zip'
+require 'ruby_dig'
 
 require 'datapackage/version'
-require 'datapackage/validator'
-require 'datapackage/package'
+require 'datapackage/exceptions'
+require 'datapackage/schema'
+require 'datapackage/resource'
+require 'datapackage/package'
+require 'datapackage/registry'

metadata

@@ -1,116 +1,216 @@
 --- !ruby/object:Gem::Specification
 name: datapackage
 version: !ruby/object:Gem::Version
-  version: 0.0.4
+  version: 0.1.0
 platform: ruby
 authors:
 - Leigh Dodds
+- pezholio
+- pikesley
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-06 00:00:00.000000000 Z
+date: 2016-09-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.8.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 1.8.3
 - !ruby/object:Gem::Dependency
   name: json-schema
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rest-client
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: colorize
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: rack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.6.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.6.4
+- !ruby/object:Gem::Dependency
+  name: rubyzip
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby_dig
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: simplecov-rcov
+  name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: fakeweb
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: henry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: 
 email:
-- leigh@ldodds.com
+- ops@theodi.org
 executables:
 - datapackage
 extensions: []
@@ -119,16 +219,16 @@ files:
 - LICENSE.md
 - README.md
 - bin/datapackage
-- etc/README.md
-- etc/csvddf-dialect-schema.json
-- etc/datapackage-schema.json
-- etc/jsontable-schema.json
 - lib/datapackage.rb
+- lib/datapackage/exceptions.rb
 - lib/datapackage/package.rb
-- lib/datapackage/validator.rb
+- lib/datapackage/registry.rb
+- lib/datapackage/resource.rb
+- lib/datapackage/schema.rb
 - lib/datapackage/version.rb
 homepage: http://github.com/theodi/datapackage.rb
-licenses: []
+licenses:
+- MIT
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -136,17 +236,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: '2.0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.2.0
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Library for working with data packages

data/etc/README.md

@@ -1,18 +0,0 @@
-This directory contains some JSON Schema documents for validating:
-
-* `datapackage-schema.json` -- [datapackage.json](http://dataprotocols.org/data-packages/) package files
-* `jsontable-schema.json` -- [JSON Table Schemas](http://dataprotocols.org/json-table-schema/) objects
-* `csvddf-dialect-schema.json` -- [CSV Dialect Description Format](http://dataprotocols.org/csv-dialect/) dialect objects
-
-The JSON Table Schemas and CSV Dialect Description Format both define JSON object structures that can appear in `datapackage.json` files (via the `schema` and `dialect` keywords). In the main `datapackage-schema.json` object, these keywords are only validated as simple objects. 
-
-In the application the subsidiary schemas are automatically applied to relevant keys. This could be improved by using JSON Schema cross-referencing.
-
-Other potential improvements include:
-
-* Add `data` keyword validation to `datapackage-schema.json`
-* Add `format` keywords for validating email addresses and date/date-times
-* Or, add `pattern` for validating dates
-* Improve regexs used in various places
-
-

data/etc/csvddf-dialect-schema.json

@@ -1,24 +0,0 @@
-{
-    "$schema": "http://json-schema.org/draft-04/schema#",
-    "title": "CSVDDF",
-    "description": "JSON Schema for validating CSVDDF dialect structures",
-    "type": "object",
-    "properties": {
-        "delimiter": {
-            "type": "string"
-        },
-        "doublequote": {
-            "type": "boolean"
-        },
-        "lineterminator": {
-            "type": "string"
-        },
-        "quotechar": {
-            "type": "string"
-        },
-        "skipinitialspace": {
-            "type": "boolean"
-        }
-    },
-    "required": [ "delimiter", "doublequote", "lineterminator", "quotechar", "skipinitialspace" ]
-}