datapackage 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: d07ade783e277a0c0f4559c64435201036560e30
-  data.tar.gz: dc64a0a4b573ccbbd376e1f2c8d47bcceb1b0475
+SHA256:
+  metadata.gz: c75abeabc9ffe3f12589976c2790dec6c2e4a748b97a99d196408fd02689d432
+  data.tar.gz: 94deaf207fd46c81e154b0617cbfcd2074f10dce2243eff888bb414618f756b8
 SHA512:
-  metadata.gz: ba0539cd939c96123bd11da407171f07a720a9dd9a2de96e86f77e8e62a243b0272a04609838aaeb895bb421dbe04303110bfa416aacd0b5cf69b03badad0858
-  data.tar.gz: c93372165ca7b145843912115c1d98eeceb9ea8dd8826b85e2702e2f5cd343fc5cfc4300ce30716b1110aee987f753adb1eaae3837011495e985064f0a9fd3b1
+  metadata.gz: 2a54d0a226c49d9a38db69f0a48f5e56220a8636ff5c27601a56d5d4325f3a0efbcb5999347631c4e0238bc4429dcf532391160decc3a4179703c05afb6c343b
+  data.tar.gz: 6390e3c8018ef3c6b8bf2c654b32e51dee41bca12762c6bc45ee5002547aa4080786c4235a331e0794458b855b884bd991522eb0621cd6fe6f1fc6e8d55dd5f5

data/lib/datapackage.rb

@@ -16,3 +16,4 @@ require 'datapackage/profile'
 require 'datapackage/resource'
 require 'datapackage/package'
 require 'datapackage/registry'
+require 'datapackage/interpreter'

data/lib/datapackage/interpreter.rb

@@ -0,0 +1,66 @@
+module DataPackage
+  class Interpreter
+    INFER_THRESHOLD = 10
+    INFER_CONFIDENCE = 0.75
+    YEAR_PATTERN = /[12]\d{3}/
+    DATE_PATTERN = /(\d{1,2}[-\/]\d{1,2}[-\/]\d{2,4})|(\d{4}[-\/]\d{1,2}[-\/]\d{1,2})/
+    DATETIME_PATTERN = /(\d{1,2}[-\/]\d{1,2}[-\/]\d{2,4}|\d{4}[-\/]\d{1,2}[-\/]\d{1,2}).\d{1,2}:\d{2}/
+    TIME_PATTERN = /^\d{1,2}((:\d{1,2})|(am|pm|AM|PM))$/
+    INTEGER_PATTERN = /^\d+$/
+    DEFAULT_TYPE_FORMAT = {'type' => 'any', 'format' => 'default'}
+
+    attr_reader :csv, :threshold
+
+    def initialize(csv)
+      @csv = csv
+      @threshold = [csv.length, INFER_THRESHOLD].min
+    end
+
+    def type_and_format_at(header)
+      values = csv.values_at(header).flatten
+      counter = {}
+      type_and_format = DEFAULT_TYPE_FORMAT
+
+      values.each_with_index do |value, i|
+        inspection_count = i + 1
+
+        inspection = inspect_value(value)
+        counter[inspection] = (counter[inspection] || 0) + 1
+        if inspection_count >= threshold
+          if counter[inspection] / inspection_count >= INFER_CONFIDENCE
+            type_and_format = inspection
+            break
+          end
+        end
+      end
+
+      type_and_format
+    end
+
+    def inspect_value(value)
+      return DEFAULT_TYPE_FORMAT unless value.is_a?(String)
+
+      if value.length == 4 && value.match(YEAR_PATTERN)
+        return { 'type' => 'year', 'format' => 'default' }
+      end
+
+      if value.match(DATETIME_PATTERN)
+        return { 'type' => 'datetime', 'format' => 'default' }
+      end
+
+      if value.match(DATE_PATTERN)
+        return { 'type' => 'date', 'format' => 'default' }
+      end
+
+      if value.match(TIME_PATTERN)
+        return { 'type' => 'time', 'format' => 'default' }
+      end
+
+      if value.match(INTEGER_PATTERN)
+        return { 'type' => 'integer', 'format' => 'default' }
+      end
+
+      DEFAULT_TYPE_FORMAT
+    end
+  end
+end

data/lib/datapackage/package.rb

@@ -119,6 +119,24 @@ module DataPackage
       self[property] || default
     end
 
+    def infer(base_path: nil, directory: nil)
+      raise PackageException.new('Base path is required for infer') unless base_path
+      raise PackageException.new('Directory is required for infer') unless directory
+
+      dir_path = File.join(base_path, directory)
+      Dir.glob("#{dir_path}/*.csv") do |filename|
+        resource = Resource.infer(filename)
+        add_resource(resource)
+      end
+
+      # If there were CSVs, this is a tabular data package
+      if resources.count > 0
+        self['profile'] = 'tabular-data-package'
+      end
+
+      descriptor
+    end
+
     # Private
 
     private

data/lib/datapackage/resource.rb

@@ -6,6 +6,33 @@ module DataPackage
 
     attr_reader :errors, :profile, :name, :source
 
+    def self.infer(filepath)
+      name = File.basename(filepath)
+      if name[-4..-1] != '.csv'
+        raise ResourceException.new('Inferrable resource must have .csv extension')
+      end
+
+      descr = {
+        'format' => 'csv',
+        'mediatype' => 'text/csv',
+        'name' => name[0...-4],
+        'path' => filepath,
+        'schema' => {
+          'fields' => []
+        },
+      }
+
+      csv = CSV.read(filepath, headers: true)
+      interpreter = DataPackage::Interpreter.new(csv)
+      csv.headers.each do |header|
+        field = { 'name' => header, 'type' => 'string'}
+        field.merge! interpreter.type_and_format_at(header)
+        descr['schema']['fields'] << field
+      end
+
+      new(descr)
+    end
+
     def initialize(resource, base_path = '')
       self.merge! dereference_descriptor(resource, base_path: base_path,
         reference_fields: ['schema', 'dialect'])
@@ -132,6 +159,7 @@ module DataPackage
     end
 
     def apply_table_defaults!
+      self['profile'] = DataPackage::DEFAULTS[:resource][:tabular_profile]
       if self.fetch('schema', nil)
         self['schema']['missingValues'] = DataPackage::DEFAULTS[:schema][:missing_values]
         self['schema'].fetch('fields', []).each do |field_descriptor|

data/lib/datapackage/version.rb

@@ -1,3 +1,3 @@
 module DataPackage
-  VERSION = "1.0.0"
+  VERSION = "1.1.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: datapackage
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Leigh Dodds
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-09-04 00:00:00.000000000 Z
+date: 2019-11-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json-schema
@@ -183,6 +183,7 @@ files:
 - lib/datapackage/defaults.rb
 - lib/datapackage/exceptions.rb
 - lib/datapackage/helpers.rb
+- lib/datapackage/interpreter.rb
 - lib/datapackage/package.rb
 - lib/datapackage/profile.rb
 - lib/datapackage/registry.rb
@@ -215,7 +216,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.11
+rubygems_version: 2.7.7
 signing_key: 
 specification_version: 4
 summary: Library for working with data packages