tableschema 0.3.1 → 0.4.0

data/lib/tableschema/model.rb

@@ -1,73 +1,96 @@
+require 'tableschema/defaults'
+
 module TableSchema
   module Model
 
-    DEFAULTS = {
-      'format' => 'default',
-      'type' => 'string'
-    }
-
     def headers
-      fields.map { |f| transform(f['name']) }
+      fields.map { |f| transform(f[:name]) }
     rescue NoMethodError
       []
     end
 
+    alias :field_names :headers
+
     def fields
-      self['fields']
+      self[:fields]
     end
 
     def primary_keys
-      [self['primaryKey']].flatten.reject { |k| k.nil? }
+      [self[:primaryKey]].flatten.reject { |k| k.nil? }
     end
 
     def foreign_keys
-      self['foreignKeys'] || []
+      self[:foreignKeys] || []
+    end
+
+    def missing_values
+      self.fetch(:missingValues, TableSchema::DEFAULTS[:missing_values])
     end
 
-    def get_type(key)
-      get_field(key)['type']
+    def get_type(field_name)
+      get_field(field_name)[:type]
     end
 
-    def get_constraints(key)
-      get_field(key)['constraints'] || {}
+    def get_constraints(field_name)
+      get_field(field_name)[:constraints] || {}
     end
 
     def required_headers
-      fields.select { |f| f['constraints']!= nil && f['constraints']['required'] == true }
-            .map { |f| transform(f['name']) }
-    rescue NoMethodError
-      []
+      fields.select { |f| f.fetch(:constraints, {}).fetch(:required, nil).to_s == 'true' }
+            .map { |f| transform(f[:name]) }
+    end
+
+    def unique_headers
+      fields.select { |f| f.fetch(:constraints, {}).fetch(:unique, nil).to_s == 'true' }
+            .map { |f| transform(f[:name]) }
     end
 
-    def has_field?(key)
-      get_field(key) != nil
+    def has_field?(field_name)
+      get_field(field_name) != nil
     end
 
-    def get_field(key)
-      fields.find { |f| f['name'] == key }
+    def get_field(field_name)
+      fields.find { |f| f[:name] == field_name }
     end
 
     def get_fields_by_type(type)
-      fields.select { |f| f['type'] == type }
+      fields.select { |f| f[:type] == type }
+    end
+
+    def add_field(descriptor)
+      self[:fields].push(descriptor)
+      validate!
+      descriptor
+    rescue TableSchema::SchemaException => e
+      self[:fields].pop
+      raise e if @strict
+      nil
+    end
+
+    def remove_field(field_name)
+      field = get_field(field_name)
+      self[:fields].reject!{ |f| f.name == field_name }
+      validate
+      field
     end
 
     private
 
-      def transform(name)
-        name.downcase! if @opts[:case_insensitive_headers]
-        name
-      end
+    def transform(name)
+      name.downcase! if @case_insensitive_headers == true
+      name
+    end
 
-      def expand!
-        (self['fields'] || []).each do |f|
-          f['type'] = DEFAULTS['type'] if f['type'] == nil
-          f['format'] = DEFAULTS['format'] if f['format'] == nil
-        end
+    def expand!
+      (self[:fields] || []).each do |f|
+        f[:type] = TableSchema::DEFAULTS[:type] if f[:type] == nil
+        f[:format] = TableSchema::DEFAULTS[:format] if f[:format] == nil
       end
+    end
 
-      def load_fields!
-        self['fields'] = (self['fields'] || []).map { |f| TableSchema::Field.new(f) }
-      end
+    def load_fields!
+      self[:fields] = (self[:fields] || []).map { |f| TableSchema::Field.new(f, missing_values) }
+    end
 
   end
 end

data/lib/tableschema/schema.rb

@@ -2,16 +2,23 @@ module TableSchema
   class Schema < Hash
     include TableSchema::Validate
     include TableSchema::Model
-    include TableSchema::Data
     include TableSchema::Helpers
 
-    def initialize(descriptor, opts = {})
-      self.merge! parse_schema(descriptor)
-      @messages = []
-      @opts = opts
+    attr_reader :errors
+
+    def initialize(descriptor, case_insensitive_headers: false, strict: false)
+      self.merge! deep_symbolize_keys(parse_schema(descriptor))
+      @case_insensitive_headers = case_insensitive_headers
+      @strict = strict
       load_fields!
       load_validator!
       expand!
+      @strict == true ? validate! : validate
+      self
+    end
+
+    def descriptor
+      self.to_h
     end
 
     def parse_schema(descriptor)
@@ -19,7 +26,7 @@ module TableSchema
         descriptor
       elsif descriptor.class == String
         begin
-          JSON.parse open(descriptor).read
+          JSON.parse(open(descriptor).read, symbolize_names: true)
         rescue Errno::ENOENT
           raise SchemaException.new("File not found at `#{descriptor}`")
         rescue OpenURI::HTTPError => e
@@ -32,5 +39,32 @@ module TableSchema
       end
     end
 
+    def cast_row(row, fail_fast: true)
+      errors = Set.new
+      handle_error = lambda { |e| fail_fast == true ? raise(e) : errors << e }
+      row = row.fields if row.class == CSV::Row
+      if row.count != self.fields.count
+        handle_error.call(TableSchema::ConversionError.new("The number of items to convert (#{row.count}) does not match the number of headers in the schema (#{self.fields.count})"))
+      end
+
+      self.fields.each_with_index do |field, i|
+        begin
+          row[i] = field.cast_value(row[i])
+        rescue TableSchema::Exception => e
+          handle_error.call(e)
+        end
+      end
+
+      unless errors.empty?
+        raise(TableSchema::MultipleInvalid.new("There were errors parsing the data", errors))
+      end
+      row
+    end
+
+    def save(target)
+      File.open(target, "w") { |file| file << JSON.pretty_generate(self.descriptor) }
+      true
+    end
+
   end
 end

data/lib/tableschema/table.rb

@@ -1,51 +1,100 @@
 module TableSchema
   class Table
 
-    attr_reader :schema
+    attr_reader :schema, :headers
 
-    def self.infer_schema(csv, opts = {})
-      TableSchema::Table.new(csv, nil, opts)
+    def self.infer_schema(csv, csv_options: {})
+      TableSchema::Table.new(csv, nil, csv_options)
     end
 
-    def initialize(csv, descriptor, opts = {})
-      @opts = opts
+    def initialize(csv, descriptor, csv_options: {})
+      @csv_options = csv_options.merge(headers: true)
       @csv = parse_csv(csv)
-      @schema = descriptor.nil? ? infer_schema(@csv) : TableSchema::Schema.new(descriptor)
+      @headers = initialize_headers
+      @schema = descriptor.nil? ? infer_schema : TableSchema::Schema.new(descriptor)
+      initialize_unique_colums
     end
 
+    def iter(row_limit: nil, cast: true, keyed: false)
+      unless block_given?
+        return enum_for(:iter, row_limit: row_limit, cast: cast, keyed: keyed)
+      end
+
+      @csv.each_with_index do |row, i|
+        break if row_limit && (row_limit <= i)
+        if cast == true
+          cast_values = @schema.cast_row(row)
+          row = CSV::Row.new(@headers, cast_values)
+          check_unique_fields(row, i)
+        end
+        if keyed == true
+          yield row.to_h
+        else
+          yield row.fields
+        end
+        collect_unique_fields(row, i)
+      end
+
+      @csv.rewind
+    end
+
+    def read(row_limit: nil, cast: true, keyed: false)
+      iterator = self.iter(row_limit: row_limit, cast: cast, keyed: keyed)
+      iterator.to_a
+    end
+
+    def save(target)
+      CSV.open(target, "wb", @csv_options) do |csv|
+        csv << @headers
+        self.iter{ |row| csv << row }
+      end
+      true
+    end
+
+    private
+
     def parse_csv(csv)
       csv = csv.is_a?(Array) ? StringIO.new(array_to_csv csv) : open(csv)
-      CSV.new(csv, csv_options)
+      CSV.new(csv, @csv_options)
     end
 
-    def csv_options
-      (@opts[:csv_options] || {}).merge(headers: true)
+    def array_to_csv(array)
+      array.map { |row| row.to_csv(row_sep: nil) }.join("\r\n")
     end
 
-    def rows(opts = {})
-      fail_fast = opts[:fail_fast] || opts[:fail_fast].nil?
-      converted = @schema.cast_rows(@csv, fail_fast, opts[:limit])
-      opts[:keyed] ? coverted_to_hash(@csv.headers, converted) : converted
+    def infer_schema
+      inferer = TableSchema::Infer.new(@headers, @csv)
+      @csv.rewind
+      inferer.schema
     end
 
-    private
+    def initialize_headers
+      headers = @csv.first.to_h.keys
+      @csv.rewind
+      headers
+    end
 
-      def array_to_csv(array)
-        array.map { |row| row.to_csv(row_sep: nil) }.join("\r\n")
+    def initialize_unique_colums
+      @unique_columns = {}
+      unless @schema.unique_headers.empty?
+        @schema.unique_headers.each{ |header| @unique_columns[header] = [] }
       end
+    end
 
-      def coverted_to_hash(headers, array)
-        array.map do |row|
-          Hash[row.map.with_index { |col, i| [headers[i], col] }]
-        end
-      end
+    def collect_unique_fields(row, row_number)
+      @unique_columns.each { |col_name, values| values[row_number] = row[col_name] }
+    end
 
-      def infer_schema(csv)
-        headers = csv.first.to_h.keys
-        csv.rewind
-        inferer = TableSchema::Infer.new(headers, csv)
-        inferer.schema
+    def check_unique_fields(row, row_number)
+      @unique_columns.each do |col_name, values|
+        row_value = row[col_name]
+        previous_values = values[0..row_number-1]
+        previous_values.map!{|value| @schema.get_field(col_name).cast_type(value)}
+        if previous_values.include?(row_value)
+          raise TableSchema::ConstraintError.new("The values for the field `#{col_name}` should be unique but value `#{row_value}` is repeated")
+        end
       end
+    end
 
   end
 end

data/lib/tableschema/types/any.rb

@@ -9,6 +9,7 @@ module TableSchema
       def self.supported_constraints
         [
           'required',
+          'unique',
           'pattern',
           'enum'
         ]

data/lib/tableschema/types/array.rb

@@ -9,6 +9,7 @@ module TableSchema
       def self.supported_constraints
         [
           'required',
+          'unique',
           'pattern',
           'enum',
           'minLength',
@@ -22,7 +23,7 @@ module TableSchema
 
       def cast_default(value)
         return value if value.is_a?(type)
-        parsed = JSON.parse(value)
+        parsed = JSON.parse(value, symbolize_names: true)
         if parsed.is_a?(type)
           return parsed
         else

data/lib/tableschema/types/base.rb

@@ -1,20 +1,16 @@
+require 'tableschema/defaults'
+
 module TableSchema
   module Types
     class Base
       include TableSchema::Helpers
 
-
       def initialize(field)
         @field = field
-        @constraints = field['constraints'] || {}
-        @required = ['true', true].include?(@constraints['required'])
-        @type = @field['type']
         set_format
       end
 
-      def cast(value, skip_constraints = false)
-        TableSchema::Constraints.new(@field, value).validate! unless skip_constraints
-        return nil if is_null?(value)
+      def cast(value)
         send("cast_#{@format}", value)
       rescue NoMethodError => e
         if e.message.start_with?('undefined method `cast_')
@@ -25,30 +21,22 @@ module TableSchema
       end
 
       def test(value)
-        cast(value, true)
+        cast(value)
         true
       rescue TableSchema::Exception
         false
       end
 
+      private
+
       def set_format
-        if (@field['format'] || '').start_with?('fmt:')
-          @format, @format_string = *@field['format'].split(':', 2)
+        if (@field[:format] || '').start_with?('fmt:')
+          @format, @format_string = *@field[:format].split(':', 2)
         else
-          @format = @field['format'] || 'default'
+          @format = @field[:format] || TableSchema::DEFAULTS[:format]
         end
       end
 
-      private
-
-        def is_null?(value)
-          null_values.include?(value) && @required == false
-        end
-
-        def null_values
-          ['null', 'none', 'nil', 'nan', '-', '']
-        end
-
     end
   end
 end

data/lib/tableschema/types/date.rb

@@ -9,6 +9,7 @@ module TableSchema
       def self.supported_constraints
         [
           'required',
+          'unique',
           'pattern',
           'enum',
           'minimum',

data/lib/tableschema/types/datetime.rb

@@ -9,6 +9,7 @@ module TableSchema
       def self.supported_constraints
         [
           'required',
+          'unique',
           'pattern',
           'enum',
           'minimum',

data/lib/tableschema/types/duration.rb

@@ -0,0 +1,31 @@
+module TableSchema
+  module Types
+    class Duration < Base
+
+      def name
+        'duration'
+      end
+
+      def self.supported_constraints
+        [
+          'required',
+          'unique',
+          'enum',
+          'minimum',
+          'maximum',
+        ]
+      end
+
+      def type
+        ActiveSupport::Duration
+      end
+
+      def cast_default(value)
+        ActiveSupport::Duration.parse(value)
+      rescue ActiveSupport::Duration::ISO8601Parser::ParsingError, TypeError
+          raise TableSchema::InvalidDurationType.new("#{value} is not a valid duration")
+      end
+
+    end
+  end
+end