jsontableschema 0.1.0

data/lib/jsontableschema/helpers.rb

@@ -0,0 +1,48 @@
+module JsonTableSchema
+  module Helpers
+
+    def convert_to_boolean(value)
+      if value.is_a?(Boolean)
+        return value
+      elsif true_values.include?(value.to_s.downcase)
+        true
+      elsif false_values.include?(value.to_s.downcase)
+        false
+      else
+        nil
+      end
+    end
+
+    def true_values
+      ['yes', 'y', 'true', 't', '1']
+    end
+
+    def false_values
+      ['no', 'n', 'false', 'f', '0']
+    end
+
+    def get_class_for_type(type)
+      "JsonTableSchema::Types::#{type_class_lookup[type]}"
+    end
+
+    def type_class_lookup
+      {
+        'any' => 'Any',
+        'array' => 'Array',
+        'base' => 'Base',
+        'boolean' => 'Boolean',
+        'date' => 'Date',
+        'datetime' => 'DateTime',
+        'geojson' => 'GeoJSON',
+        'geopoint' => 'GeoPoint',
+        'integer' => 'Integer',
+        'null' => 'Null',
+        'number' => 'Number',
+        'object' => 'Object',
+        'string' => 'String',
+        'time' => 'Time',
+      }
+    end
+
+  end
+end

data/lib/jsontableschema/infer.rb

@@ -0,0 +1,142 @@
+module JsonTableSchema
+  class Infer
+
+    include JsonTableSchema::Helpers
+
+    attr_reader :schema
+
+    def initialize(headers, rows, opts = {})
+      @headers = headers
+      @rows = rows
+      @explicit = opts[:explicit]
+      @primary_key = opts[:primary_key]
+      @row_limit = opts[:row_limit]
+
+      @schema = {
+        'fields' => fields
+      }
+      @schema['primaryKey'] = @primary_key if @primary_key
+      infer!
+    end
+
+    def fields
+      @headers.map do |header|
+        descriptor = {
+          'name' => header,
+          'title' => '',
+          'description' => '',
+        }
+
+        constraints = {}
+        constraints['required'] = @explicit === true
+        constraints['unique'] = (header == @primary_key)
+        constraints.delete_if { |k,v| v == false } unless @explicit === true
+        descriptor['constraints'] = constraints if constraints.count > 0
+        descriptor
+      end
+    end
+
+    def infer!
+      type_matches = []
+      @rows.each_with_index do |row, i|
+        break if @row_limit && i > @row_limit
+
+        row_length = row.count
+        headers_length = @headers.count
+
+        if row_length > headers_length
+          row = row[0..headers_length]
+        elsif row_length < headers_length
+          diff = headers_length - row_length
+          fill = [''] * diff
+          row = row.push(fill).flatten
+        end
+
+        row.each_with_index do |col, i|
+          type_matches[i] ||= []
+          type_matches[i] << guess_type(col, i)
+        end
+
+      end
+      resolve_types(type_matches)
+      @schema = JsonTableSchema::Schema.new(@schema)
+    end
+
+    def guess_type(col, index)
+      guessed_type = 'string'
+      guessed_format = 'default'
+
+      available_types.reverse_each do |type|
+        klass = get_class_for_type(type)
+        converter = Kernel.const_get(klass).new(@schema['fields'][index])
+        if converter.test(col) === true
+          guessed_type = type
+          guessed_format = guess_format(converter, col)
+          break
+        end
+      end
+
+      {
+        'type' => guessed_type,
+        'format' => guessed_format
+      }
+    end
+
+    def guess_format(converter, col)
+      guessed_format = 'default'
+      converter.class.instance_methods.grep(/cast_/).each do |method|
+        begin
+          format = method.to_s
+          format.slice!('cast_')
+          next if format == 'default'
+          converter.send(method, col)
+          guessed_format = format
+          break
+        rescue JsonTableSchema::Exception
+        end
+      end
+      guessed_format
+    end
+
+    def resolve_types(results)
+      results.each_with_index do |result,v|
+        result.uniq!
+
+        if result.count == 1
+          rv = result[0]
+        else
+          counts = {}
+          result.each do |r|
+            counts[r] ||= 0
+            counts[r] += 1
+          end
+
+          sorted_counts = counts.sort_by {|_key, value| value}
+          rv = sorted_counts[0][0]
+        end
+
+        @schema['fields'][v].merge!(rv)
+      end
+
+    end
+
+    def available_types
+      [
+        'any',
+        'string',
+        'boolean',
+        'number',
+        'integer',
+        'null',
+        'date',
+        'time',
+        'datetime',
+        'array',
+        'object',
+        'geopoint',
+        'geojson'
+      ]
+    end
+
+  end
+end

data/lib/jsontableschema/model.rb

@@ -0,0 +1,73 @@
+module JsonTableSchema
+  module Model
+
+    DEFAULTS = {
+      'format' => 'default',
+      'type' => 'string'
+    }
+
+    def headers
+      fields.map { |f| transform(f['name']) }
+    rescue NoMethodError
+      []
+    end
+
+    def fields
+      self['fields']
+    end
+
+    def primary_keys
+      [self['primaryKey']].flatten.reject { |k| k.nil? }
+    end
+
+    def foreign_keys
+      self['foreignKeys'] || []
+    end
+
+    def get_type(key)
+      get_field(key)['type']
+    end
+
+    def get_constraints(key)
+      get_field(key)['constraints'] || {}
+    end
+
+    def required_headers
+      fields.select { |f| f['constraints']!= nil && f['constraints']['required'] == true }
+            .map { |f| transform(f['name']) }
+    rescue NoMethodError
+      []
+    end
+
+    def has_field?(key)
+      get_field(key) != nil
+    end
+
+    def get_field(key)
+      fields.find { |f| f['name'] == key }
+    end
+
+    def get_fields_by_type(type)
+      fields.select { |f| f['type'] == type }
+    end
+
+    private
+
+      def fields
+        self['fields']
+      end
+
+      def transform(name)
+        name.downcase! if @opts[:case_insensitive_headers]
+        name
+      end
+
+      def expand!
+        (self['fields'] || []).each do |f|
+          f['type'] = DEFAULTS['type'] if f['type'] == nil
+          f['format'] = DEFAULTS['format'] if f['format'] == nil
+        end
+      end
+
+  end
+end

data/lib/jsontableschema/schema.rb

@@ -0,0 +1,35 @@
+module JsonTableSchema
+  class Schema < Hash
+    include JsonTableSchema::Validate
+    include JsonTableSchema::Model
+    include JsonTableSchema::Data
+    include JsonTableSchema::Helpers
+
+    def initialize(schema, opts = {})
+      self.merge! parse_schema(schema)
+      @messages = []
+      @opts = opts
+      load_validator!
+      expand!
+    end
+
+    def parse_schema(schema)
+      if schema.class == Hash
+        schema
+      elsif schema.class == String
+        begin
+          JSON.parse open(schema).read
+        rescue Errno::ENOENT
+          raise SchemaException.new("File not found at `#{schema}`")
+        rescue OpenURI::HTTPError => e
+          raise SchemaException.new("URL `#{schema}` returned #{e.message}")
+        rescue JSON::ParserError
+          raise SchemaException.new("File at `#{schema}` is not valid JSON")
+        end
+      else
+        raise SchemaException.new("A schema must be a hash, path or URL")
+      end
+    end
+
+  end
+end

data/lib/jsontableschema/table.rb

@@ -0,0 +1,50 @@
+module JsonTableSchema
+  class Table
+
+    attr_reader :schema
+
+    def self.infer_schema(csv, opts = {})
+      JsonTableSchema::Table.new(csv, nil, opts)
+    end
+
+    def initialize(csv, schema, opts = {})
+      @opts = opts
+      @csv = parse_csv(csv)
+      @schema = schema.nil? ? infer_schema(@csv) : JsonTableSchema::Schema.new(schema)
+    end
+
+    def parse_csv(csv)
+      csv_string = csv.is_a?(Array) ? array_to_csv(csv) : open(csv).read
+      CSV.parse(csv_string, csv_options)
+    end
+
+    def csv_options
+      (@opts[:csv_options] || {}).merge(headers: true)
+    end
+
+    def rows(opts = {})
+      fail_fast = opts[:fail_fast] || opts[:fail_fast].nil?
+      rows = opts[:limit] ? @csv.to_a.drop(1).take(opts[:limit]) : @csv.to_a.drop(1)
+      converted = @schema.convert(rows, fail_fast)
+      opts[:keyed] ? coverted_to_hash(@csv.headers, converted) : converted
+    end
+
+    private
+
+      def array_to_csv(array)
+        array.map { |row| row.to_csv(row_sep: nil) }.join("\r\n")
+      end
+
+      def coverted_to_hash(headers, array)
+        array.map do |row|
+          Hash[row.map.with_index { |col, i| [headers[i], col] }]
+        end
+      end
+
+      def infer_schema(csv)
+        inferer = JsonTableSchema::Infer.new(csv.headers, csv.to_a)
+        inferer.schema
+      end
+
+  end
+end

data/lib/jsontableschema/types/any.rb

@@ -0,0 +1,23 @@
+module JsonTableSchema
+  module Types
+    class Any < Base
+
+      def name
+        'any'
+      end
+
+      def self.supported_constraints
+        [
+          'required',
+          'pattern',
+          'enum'
+        ]
+      end
+
+      def cast_default(value)
+        value
+      end
+
+    end
+  end
+end

data/lib/jsontableschema/types/array.rb

@@ -0,0 +1,37 @@
+module JsonTableSchema
+  module Types
+    class Array < Base
+
+      def name
+        'array'
+      end
+
+      def self.supported_constraints
+        [
+          'required',
+          'pattern',
+          'enum',
+          'minLength',
+          'maxLength',
+        ]
+      end
+
+      def type
+        ::Array
+      end
+
+      def cast_default(value)
+        return value if value.is_a?(type)
+        parsed = JSON.parse(value)
+        if parsed.is_a?(type)
+          return parsed
+        else
+          raise JsonTableSchema::InvalidArrayType.new("#{value} is not a valid array")
+        end
+      rescue
+        raise JsonTableSchema::InvalidArrayType.new("#{value} is not a valid array")
+      end
+
+    end
+  end
+end

data/lib/jsontableschema/types/base.rb

@@ -0,0 +1,54 @@
+module JsonTableSchema
+  module Types
+    class Base
+      include JsonTableSchema::Helpers
+
+
+      def initialize(field)
+        @field = field
+        @constraints = field['constraints'] || {}
+        @required = ['true', true].include?(@constraints['required'])
+        @type = @field['type']
+        set_format
+      end
+
+      def cast(value, skip_constraints = false)
+        JsonTableSchema::Constraints.new(@field, value).validate! unless skip_constraints
+        return nil if is_null?(value)
+        send("cast_#{@format}", value)
+      rescue NoMethodError => e
+        if e.message.start_with?('undefined method `cast_')
+          raise(JsonTableSchema::InvalidFormat.new("The format `#{@format}` is not supported by the type `#{@type}`"))
+        else
+          raise e
+        end
+      end
+
+      def test(value)
+        cast(value, true)
+        true
+      rescue JsonTableSchema::Exception
+        false
+      end
+
+      def set_format
+        if (@field['format'] || '').start_with?('fmt:')
+          @format, @format_string = *@field['format'].split(':', 2)
+        else
+          @format = @field['format'] || 'default'
+        end
+      end
+
+      private
+
+        def is_null?(value)
+          null_values.include?(value) && @required == false
+        end
+
+        def null_values
+          ['null', 'none', 'nil', 'nan', '-', '']
+        end
+
+    end
+  end
+end