masamune 0.17.6 → 0.17.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c29146a9633b2cf7760df9db790297c0889d856c
-  data.tar.gz: d655ab1bd2088a9535b132dba044fe94a9c6eaa8
+  metadata.gz: 36f6999cb7287d4d466015d50c502be5330b1b9f
+  data.tar.gz: 39ff7b7dbd08db306bd2c35d7058f5e8040b781e
 SHA512:
-  metadata.gz: bd15f96322e460a7e01c88465d4b245ee27242729c06b0567ad788fd5d02c57a72aea02b5e7cf58d9fe5217018e33b271f6723c8e522fcea37fbbb95b56afbbd
-  data.tar.gz: 79c5a0895b9a64ff4a7b0ce388fc344a05e21b10c674b1069fe671ce486e55e2eaddb8e81792f513da95dec966f4e9380b537627faa93e75f19828ebf90df414
+  metadata.gz: 3c5d779aae19fd7b39f427e151c218c3d54fa1ede5596018a485ce8e0c25a66bee9307a0c314be2a0239a9adcfd036316240a4de9d40d04285743f673fca7669
+  data.tar.gz: 99acb8d19ce4458ac58636aaa2a5f31817000d6e11df4af086e1e59c73c0bb27bf5080cc937c2941961c713b4f3714ef32390fa6dad976d3ca946f143e2fd7ba

data/lib/masamune/schema/column.rb

@@ -288,7 +288,7 @@ module Masamune::Schema
           {}
         end
       when :string
-        value.to_s
+        value.blank? ? nil : value.to_s
       else
         value
       end

data/lib/masamune/schema/map.rb

@@ -105,22 +105,20 @@ module Masamune::Schema
         raise 'must call Buffer#bind first' unless @io
         CSV.parse(JSONEncoder.new(@io, @store), options.merge(headers: @store.headers || @table.columns.keys)) do |data|
           next if data.to_s =~ /\A#/
-          yield safe_row(data)
+          row = safe_row(data)
+          yield row.to_hash if row
           @line += 1
         end
       end
 
       def append(data)
         raise 'must call Buffer#bind first' unless @io
-        row = Masamune::Schema::Row.new(parent: @table, values: data.to_hash)
+        row = safe_row(data)
+        return unless row
         write_headers = @store.headers && @line < 1
         @csv ||= CSV.new(@io, options.merge(headers: row.headers, write_headers: write_headers))
-        if row.missing_required_columns.any?
-          missing_required_column_names = row.missing_required_columns.map(&:name)
-          @map.skip_or_raise(self, row, "missing required columns '#{missing_required_column_names.join(', ')}'")
-        else
-          @csv << row.serialize if append?(row.serialize)
-        end
+        @csv << row.serialize if row.valid? && append?(row.serialize)
+      ensure
         @line += 1
       end
 
@@ -157,10 +155,10 @@ module Masamune::Schema
       end
 
       def safe_row(data)
-        row = Masamune::Schema::Row.new(parent: @table, values: data.to_hash, strict: false)
-        row.to_hash
-      rescue
-        @map.skip_or_raise(self, data, 'failed to parse')
+        Masamune::Schema::Row.new(parent: @table, values: data.to_hash, strict: @map.fail_fast)
+      rescue => e
+        @map.skip_or_raise(self, data, e.message || 'failed to parse')
+        nil
       end
 
       def append?(elem)

data/lib/masamune/schema/row.rb

@@ -36,7 +36,10 @@ module Masamune::Schema
       attr_accessor attr
     end
 
+    attr_reader :errors
+
     def initialize(opts = {})
+      @errors = Set.new
       opts.symbolize_keys!
       DEFAULT_ATTRIBUTES.merge(opts).each do |name, value|
         public_send("#{name}=", value)
@@ -54,7 +57,8 @@ module Masamune::Schema
 
     def parent=(parent)
       @parent = parent
-      normalize_values! if @parent
+      normalize! if @parent
+      validate! if @parent
     end
 
     def name(column = nil)
@@ -104,18 +108,22 @@ module Masamune::Schema
       column.sql_value(values[column.name])
     end
 
-    def missing_required_columns
-      Set.new.tap do |missing|
-        values.select do |key, value|
-          column = @columns[key]
-          missing << column if column.required_value? && value.nil?
-        end
-      end
+    def valid?
+      errors.empty?
+    end
+
+    def invalid?
+      errors.any?
+    end
+
+    def validate!
+      validate_required_columns!
+      raise errors.first if invalid?
     end
 
     private
 
-    def normalize_values!
+    def normalize!
       result = {}
       @columns = {}
       values.each do |key, value|
@@ -124,10 +132,23 @@ module Masamune::Schema
           @columns[column.compact_name] = column
           result[column.compact_name] = column.ruby_value(value)
         elsif strict
-          raise ArgumentError, "#{@values} contains undefined columns #{key}"
+          @errors << "#{@values} contains undefined columns #{key}"
         end
       end
       @values = result
     end
+
+    def validate_required_columns!
+      @errors << "missing required columns '#{missing_required_columns.map(&:name).join(', ')}'" if missing_required_columns.any?
+    end
+
+    def missing_required_columns
+      Set.new.tap do |missing|
+        values.select do |key, value|
+          column = @columns[key]
+          missing << column if column.required_value? && value.nil?
+        end
+      end
+    end
   end
 end

data/lib/masamune/version.rb

@@ -21,5 +21,5 @@
 #  THE SOFTWARE.
 
 module Masamune
-  VERSION = '0.17.6'
+  VERSION = '0.17.7'
 end

data/spec/masamune/schema/column_spec.rb

@@ -364,6 +364,30 @@ describe Masamune::Schema::Column do
       end
     end
 
+    context 'with type :string' do
+      let(:column) { described_class.new(id: 'string', type: :string) }
+
+      context 'when nil' do
+        let(:value) { nil }
+        it { is_expected.to be(nil) }
+      end
+
+      context 'when blank' do
+        let(:value) { '' }
+        it { is_expected.to be(nil) }
+      end
+
+      context 'when String' do
+        let(:value) { 'value' }
+        it { is_expected.to eq('value') }
+      end
+
+      context 'when Integer' do
+        let(:value) { '1' }
+        it { is_expected.to eq('1') }
+      end
+    end
+
     context 'with type :timestamp' do
       let(:column) { described_class.new(id: 'timestamp', type: :timestamp) }
 

data/spec/masamune/schema/dimension_spec.rb

@@ -80,7 +80,7 @@ describe Masamune::Schema::Dimension do
         ]
     end
 
-    it { expect { dimension }.to raise_error ArgumentError, /contains undefined columns/ }
+    it { expect { dimension }.to raise_error /contains undefined columns/ }
   end
 
   context 'for type :four' do

data/spec/masamune/schema/map_spec.rb

@@ -176,22 +176,20 @@ describe Masamune::Schema::Map do
       end
 
       before do
-        expect(environment.logger).to receive(:warn).with(/missing required columns 'user_id'/).ordered
+        expect(environment.logger).to receive(:warn).with(/missing required columns 'id'/).ordered
         expect(environment.logger).to receive(:debug).with(
-          :message => %q(missing required columns 'user_id'),
+          :message => %q(missing required columns 'id'),
           :source  => 'user_stage',
           :target  => 'user_dimension_ledger',
-          :file    => output.path,
+          :file    => input.path,
           :line    => 3,
           :row     => {
-            'tenant_id'                  => 50,
-            'user_id'                    => nil,
-            'user_account_state.name'    => 'active',
-            'hr_user_account_state.name' => 'active',
-            'admin'                      => false,
-            'preferences'                => {},
-            'source'                     => 'users_file',
-            'cluster_id'                 => 100
+            'id'          => nil,
+            'tenant_id'   => '50',
+            'junk_id'     => 'X',
+            'deleted_at'  => nil,
+            'admin'       => '0',
+            'preferences' => nil
           }
         ).ordered
       end
@@ -253,9 +251,9 @@ describe Masamune::Schema::Map do
             "deleted_at"  => nil
           }
         ).ordered
-        expect(environment.logger).to receive(:warn).with(/failed to parse/).ordered
+        expect(environment.logger).to receive(:warn).with("Could not coerce 'INVALID_JSON' into :json for column 'preferences'").ordered
         expect(environment.logger).to receive(:debug).with(
-          :message => "failed to parse",
+          :message => "Could not coerce 'INVALID_JSON' into :json for column 'preferences'",
           :source  => "input_stage",
           :target  => "user_dimension_ledger",
           :file    => input.path,

data/spec/masamune/schema/table_spec.rb

@@ -135,7 +135,7 @@ describe Masamune::Schema::Table do
         ]
     end
 
-    it { expect { table }.to raise_error ArgumentError, /contains undefined columns/ }
+    it { expect { table }.to raise_error /contains undefined columns/ }
   end
 
   context 'with partial values' do

metadata

@@ -1,195 +1,195 @@
 --- !ruby/object:Gem::Specification
 name: masamune
 version: !ruby/object:Gem::Version
-  version: 0.17.6
+  version: 0.17.7
 platform: ruby
 authors:
 - Michael Andrews
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-25 00:00:00.000000000 Z
+date: 2016-03-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: thor
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: tzinfo
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: chronic
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: tilt
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: erubis
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: parallel
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rb-readline
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0.9'
 - !ruby/object:Gem::Dependency
   name: user_agent_parser
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>'
+    - - ">"
       - !ruby/object:Gem::Version
         version: '2.99'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>'
+    - - ">"
       - !ruby/object:Gem::Version
         version: '2.99'
 - !ruby/object:Gem::Dependency
   name: byebug
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Hybrid Data & Work Flow
@@ -426,17 +426,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.8
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Hybrid Data & Work Flow