datapackage 0.2.5 → 0.2.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0e72a49acec8bb887ed72db7645838841b31873f
-  data.tar.gz: ff5d34e0d9a62bc6209d57a06c76fed750395f4a
+  metadata.gz: 18b6d480843853daa822de909d242c78b40ed762
+  data.tar.gz: 4c6004ea6559656434a5759038cc0163c0762520
 SHA512:
-  metadata.gz: 84486c5bf44e5c2545014efddd2e44f2032e1108b8e2e9c24f708ec8011a212f86dd52247584e9caf6a8545b8c1aac98047877d6f6eaa0a3703b7242c71eab72
-  data.tar.gz: f8b6b0492986cc2b1fc85675a498963b4710fcb9e0337923342c57a4cf2dc001b162cfa7efb72c3928d5194e417aa71f1c545e32ffcc5e0038449531b78604d5
+  metadata.gz: e3c5286265b393c7c7ecd3563515bccfb67f2c20a54b6c15e2d4ad1ac00f6c55b69f18c591e38397958a6bf23cd9d0c88deda81ae39e06cf6d8532607f31eeda
+  data.tar.gz: d09058baf930fda7eb99d688f9c9fb405bc1910563d0b1e48763400dc877c27a1540c69eb24e292827e511500612156b37022cbf067622bb2d649fa3e9e9f84b

data/lib/datapackage.rb

@@ -0,0 +1,18 @@
+require 'date'
+require 'uri'
+require 'net/http'
+require 'csv'
+require 'json'
+require 'json-schema'
+require 'zip'
+require 'ruby_dig'
+require 'tableschema'
+
+require 'datapackage/defaults'
+require 'datapackage/helpers'
+require 'datapackage/version'
+require 'datapackage/exceptions'
+require 'datapackage/profile'
+require 'datapackage/resource'
+require 'datapackage/package'
+require 'datapackage/registry'

data/lib/datapackage/defaults.rb

@@ -0,0 +1,27 @@
+module DataPackage
+  DEFAULTS = {
+    resource: {
+      profile: 'data-resource',
+      tabular_profile: 'tabular-data-resource',
+      encoding: 'utf-8',
+    },
+    package: {
+      profile: 'data-package',
+    },
+    schema: {
+      format: 'default',
+      type: 'string',
+      missing_values: [''],
+    },
+    dialect: {
+      delimiter: ',',
+      doubleQuote: true,
+      lineTerminator: '\r\n',
+      quoteChar: '"',
+      escapeChar: '\\',
+      skipInitialSpace: true,
+      header: true,
+      caseSensitiveHeader: false,
+    },
+  }.freeze
+end

data/lib/datapackage/exceptions.rb

@@ -0,0 +1,8 @@
+module DataPackage
+  class Exception < ::Exception; end
+  class RegistryException < Exception; end
+  class ResourceException < Exception; end
+  class ProfileException < Exception; end
+  class PackageException < Exception; end
+  class ValidationError < Exception; end
+end

data/lib/datapackage/helpers.rb

@@ -0,0 +1,98 @@
+module DataPackage
+  module Helpers
+
+    # Dereference a resource that can be a URL or path to a JSON file or a hash
+    # Returns a Hash with all values that are URLs or paths dereferenced
+    def dereference_descriptor(resource, base_path: nil, reference_fields: nil)
+      options = {
+        base_path: base_path,
+        reference_fields: reference_fields,
+      }
+      case resource
+      when Hash
+        resource.inject({}) do |new_resource, (key, val)|
+          if reference_fields.nil? || reference_fields.include?(key)
+            new_resource[key] = dereference_descriptor(val, **options)
+          else
+            new_resource[key] = val
+          end
+          new_resource
+        end
+      when Enumerable
+        resource.map{ |el| dereference_descriptor(el, **options)}
+      when String
+        begin
+          resolve_json_reference(resource, deep_dereference: true, base_path: base_path)
+        rescue Errno::ENOENT
+          resource
+        end
+      else
+        resource
+      end
+    end
+
+    # Resolve a reference to a JSON file; Returns the JSON as hash
+    # Raises JSON::ParserError, OpenURI::HTTPError, SocketError or TypeError for invalid references
+    def resolve_json_reference(reference, deep_dereference: false, base_path: nil)
+      # Try to extract JSON from file or webpage
+      reference = join_paths(base_path, reference)
+      resolved_reference = load_json(reference)
+      if deep_dereference == true
+        dereference_descriptor(resolved_reference, base_path: base_path)
+      else
+        resolved_reference
+      end
+    end
+
+    # Load JSON from path or URL;
+    # Raises: Errno::ENOENT, OpenURI::HTTPError, SocketError, JSON::ParserError
+    def load_json(reference)
+      JSON.parse open(reference).read
+    end
+
+    def base_path(path_or_url)
+      path_or_url = path_or_url.to_s
+      if path_or_url.empty?
+        nil
+      elsif path_or_url =~ /\A#{URI::regexp}\z/
+        uri = URI.parse path_or_url
+        return "#{uri.scheme}://#{uri.host}#{File.dirname uri.path}".chomp('/')
+      else
+        if File.directory?(path_or_url)
+          return path_or_url
+        else
+          return File.expand_path File.dirname path_or_url
+        end
+      end
+    end
+
+    def join_paths(base_path, resource)
+      if base_path.nil? || base_path.empty?
+        resource
+      elsif base_path =~ /\A#{URI::regexp}\z/
+        URI.join(base_path, resource).to_s
+      elsif File.directory?(base_path)
+        File.join(base_path, resource).to_s
+      elsif File.file?(base_path)
+        base_path
+      else
+        resource
+      end
+    end
+
+    def is_fully_qualified_url?(string)
+      uri = URI.parse(string)
+      uri.is_a?(URI::HTTP) && !uri.host.nil?
+    rescue URI::InvalidURIError
+      false
+    end
+
+    def is_safe_path?(string)
+      path = Pathname.new(string)
+      return false if path.absolute?
+      return false unless /^\.+$/.match(path.to_s.split('/').first).nil?
+      true
+    end
+
+  end
+end

data/lib/datapackage/package.rb

@@ -0,0 +1,212 @@
+require 'open-uri'
+
+module DataPackage
+  class Package < Hash
+    include DataPackage::Helpers
+
+    attr_reader :opts, :errors, :profile, :dead_resources
+
+    # Parse or create a data package
+    # Supports reading data from JSON file, directory, and a URL
+    # descriptor:: Hash or String
+    # opts:: Options used to customize reading and parsing
+    def initialize(descriptor = nil, opts: {})
+      @opts = opts
+      @dead_resources = []
+      self.merge! parse_package(descriptor)
+      @profile = DataPackage::Profile.new(self.fetch('profile', DataPackage::DEFAULTS[:package][:profile]))
+      self['profile'] = @profile.name
+      define_properties!
+      load_resources!
+    rescue OpenURI::HTTPError, SocketError => e
+      raise PackageException.new "Package URL returned #{e.message}"
+    rescue JSON::ParserError
+      raise PackageException.new 'Package descriptor is not valid JSON'
+    end
+
+    def descriptor
+      self.to_h
+    end
+
+    # Returns the directory for a local file package or base url for a remote
+    # Returns nil for an in-memory object (because it has no base as yet)
+    def base
+      # user can override base
+      return @opts[:base] if @opts[:base]
+      return '' unless @location
+      # work out base directory or uri
+      if local?
+          return File.dirname(@location)
+      else
+          return @location.split('/')[0..-2].join('/')
+      end
+    end
+
+    # Is this a local package? Returns true if created from an in-memory object or a file/directory reference
+    def local?
+      return @local if @local
+      return false if @location =~ /\A#{URI::regexp}\z/
+      true
+    end
+
+    def resources
+      update_resources!
+      self['resources']
+    end
+
+    def resource_names
+      update_resources!
+      self['resources'].map{|res| res.name}
+    end
+
+    def valid?
+      return false unless @profile.valid?(self)
+      return false if self['resources'].map{ |resource| resource.valid? }.include?(false)
+      true
+    end
+
+    alias :valid  :valid?
+
+    def validate
+      @profile.validate(self)
+      self['resources'].each { |resource| resource.validate }
+      true
+    end
+
+    def iter_errors
+      errors = @profile.iter_errors(self){ |err| err }
+      self['resources'].each do |resource|
+        resource.iter_errors{ |err| errors << err }
+      end
+      errors.each{ |error| yield error }
+    end
+
+    def add_resource(resource)
+      resource = load_resource(resource)
+      self['resources'].push(resource)
+      begin
+        self.validate
+        resource
+      rescue DataPackage::ValidationError
+        self['resources'].pop
+        nil
+      end
+    end
+
+    def remove_resource(resource_name)
+      update_resources!
+      resource = get_resource(resource_name)
+      self['resources'].reject!{ |resource| resource.name == resource_name }
+      resource
+    end
+
+    def get_resource(resource_name)
+      update_resources!
+      self['resources'].find{ |resource| resource.name == resource_name }
+    end
+
+    def save(target=@location)
+      update_resources!
+      File.open(target, "w") { |file| file << JSON.pretty_generate(self) }
+      true
+    end
+
+    def property(property, default = nil)
+      self[property] || default
+    end
+
+    private
+
+    def define_properties!
+      (@profile['properties'] || {}).each do |k, v|
+        next if k == 'resources' || k == 'profile'
+        define_singleton_method("#{k.to_sym}=", proc { |p| set_property(k, p) })
+        define_singleton_method(k.to_sym.to_s, proc { property k, default_value(v) })
+      end
+    end
+
+    def load_resources!
+      self['resources'] ||= []
+      update_resources!
+    end
+
+    def update_resources!
+      self['resources'].map! do |resource|
+        begin
+          load_resource(resource)
+        rescue ResourceException
+          @dead_resources << resource
+          nil
+        end
+      end.compact!
+    end
+
+    def load_resource(resource)
+      if resource.is_a?(Resource)
+        resource
+      else
+        Resource.new(resource, base)
+      end
+    end
+
+    def default_value(field_data)
+      case field_data['type']
+      when 'array'
+          []
+      when 'object'
+          {}
+      else
+        nil
+      end
+    end
+
+    def set_property(key, value)
+      self[key] = value
+    end
+
+    def parse_package(descriptor)
+      # TODO: base directory/url
+      if descriptor.nil?
+        {}
+      elsif descriptor.class == Hash
+        descriptor
+      else
+        read_package(descriptor)
+      end
+    end
+
+    def read_package(descriptor)
+      if File.extname(descriptor) == '.zip'
+        unzip_package(descriptor)
+      else
+        @location = descriptor.to_s
+        load_json(descriptor)
+      end
+    end
+
+    def unzip_package(descriptor)
+      descriptor = write_to_tempfile(descriptor) if descriptor =~ /\A#{URI::regexp}\z/
+      dir = Dir.mktmpdir
+      package = {}
+      Zip::File.open(descriptor) do |zip_file|
+          # Extract all the files
+          zip_file.each { |entry| entry.extract("#{dir}/#{File.basename entry.name}") }
+          # Get and parse the datapackage metadata
+          entry = zip_file.glob("*/#{@opts[:default_filename] || 'datapackage.json'}").first
+          package = JSON.parse(entry.get_input_stream.read)
+      end
+      # Set the base dir to the directory we unzipped to
+      @opts[:base] = dir
+      # This is now a local file, not a URL
+      @local = true
+      package
+    end
+
+    def write_to_tempfile(url)
+      tempfile = Tempfile.new('datapackage')
+      tempfile.write(open(url).read)
+      tempfile.rewind
+      tempfile
+    end
+  end
+end

data/lib/datapackage/profile.rb

@@ -0,0 +1,62 @@
+module DataPackage
+  class Profile < Hash
+    include DataPackage::Helpers
+
+    attr_reader :name, :registry
+
+    def initialize(descriptor)
+      unless descriptor.is_a?(String)
+        raise ProfileException.new 'Profile must be a URL or registry identifier'
+      end
+      @name = descriptor
+      if is_fully_qualified_url?(descriptor)
+        self.merge!(load_json(descriptor))
+      else
+        self.merge!(get_profile_from_registry(descriptor))
+      end
+    rescue OpenURI::HTTPError, SocketError => e
+      raise ProfileException.new "Profile URL returned #{e.message}"
+    rescue JSON::ParserError
+      raise ProfileException.new 'Profile is not valid JSON'
+    end
+
+    def jsonschema
+      self.to_h
+    end
+
+    # Returns true if there are no errors in data, false if there are
+    def valid?(data)
+      JSON::Validator.validate(self, data)
+    end
+
+    alias :valid :valid?
+
+    # Validate data against this profile. Returns true or raises DataPackage::ValidationError
+    def validate(data)
+      JSON::Validator.validate!(self, data)
+    rescue JSON::Schema::ValidationError => e
+      raise DataPackage::ValidationError.new(e.message)
+    end
+
+    # Lazily yields each ValidationError raised for data
+    def iter_errors(data)
+      JSON::Validator.fully_validate(self, data).each{ |error| yield error }
+    end
+
+    private
+
+    def get_profile_from_registry(descriptor)
+      @registry = DataPackage::Registry.new
+      profile_metadata = registry.profiles.fetch(descriptor)
+      if profile_metadata.fetch('schema_path', nil)
+        profile_path = join_paths(base_path(registry.path), profile_metadata['schema_path'])
+      else
+        profile_path = profile_metadata['schema']
+      end
+      load_json(profile_path)
+    rescue KeyError
+      raise ProfileException.new "Couldn't find profile with id `#{descriptor}` in registry"
+    end
+
+  end
+end

data/lib/datapackage/registry.rb

@@ -0,0 +1,36 @@
+module DataPackage
+  # Allow loading Data Package profiles from the official registry.
+
+  class Registry
+    include DataPackage::Helpers
+
+    attr_reader :path, :profiles
+
+    DEFAULT_REGISTRY_URL = 'https://specs.frictionlessdata.io/schemas/registry.json'.freeze
+    DEFAULT_REGISTRY_PATH = File.join(File.expand_path(File.dirname(__FILE__)), '..', 'profiles', 'registry.json').freeze
+
+    def initialize
+      @path = DEFAULT_REGISTRY_PATH
+      @profiles = get_registry(DEFAULT_REGISTRY_PATH)
+    rescue Errno::ENOENT
+      raise RegistryException.new 'Registry path is not valid'
+    rescue OpenURI::HTTPError, SocketError => e
+      raise RegistryException.new "Registry URL returned #{e.message}"
+    rescue JSON::ParserError
+      raise RegistryException.new 'Registry descriptor is not valid JSON'
+    rescue KeyError
+      raise RegistryException.new 'Property `id` is mandatory for profiles'
+    end
+
+    private
+
+    def get_registry(descriptor)
+      resources = load_json(descriptor)
+      resources.reduce({}) do |registry, resource|
+        registry[resource['id']] = resource
+        registry
+      end
+    end
+
+  end
+end