gooddata_datawarehouse 0.0.3 → 0.0.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c02001e54624b97d0dc6ce317eea0141af49e6d7
-  data.tar.gz: f136f74d59e63b43be394b0cf57e99c7805a7c75
+  metadata.gz: e6c19b17e345eb5ea10f389067f689544bc49ce1
+  data.tar.gz: 2d7bae04776af43c98eccd798352954d5678e7e2
 SHA512:
-  metadata.gz: 37307c29876afa00c48e2a0402ff648d52a9037ce6f040f355d4a8cfaaf0e7b99847c2192537481f82d9daa025a94c5384b2fe4a2ccf2c019abfa5a5dc01fa45
-  data.tar.gz: 4f25f05dec8b8ac1f78f1ae75b924b5792d860228043d403e72fe5f4c48f8aeab1a7db836f17b2ba9220e373e14f3ec54c1ac7ed20eae474e9ecd1c6a5720643
+  metadata.gz: 926f231df825fdb1077d270a8944a9d2631d10f1ca15b1ee4d5a19fb31505a2325088872ff1a6a56b4598e197d90a7410cfaf15685fa0506e372e3829fb9839f
+  data.tar.gz: 20505562823f04ffb717260115bff12dd649ae0a81331012793fc370de1bc05e37ec73a6fbf7f7040d119e4576ccfc4bb7e8be5cf343bd720327f333949cdd9f

data/Gemfile

@@ -1,4 +1,7 @@
 source 'https://rubygems.org'
 
+# automatic ruby preselection
+#ruby=jruby-1.7.15
+
 # Specify your gem's dependencies in gooddata_datawarehouse.gemspec
 gemspec

data/lib/gooddata_datawarehouse/datawarehouse.rb

@@ -52,25 +52,31 @@ module GoodData
 
     def csv_to_new_table(table_name, csv_path, opts={})
       cols = create_table_from_csv_header(table_name, csv_path, opts)
-      load_data_from_csv(table_name, csv_path, columns: cols)
+      load_data_from_csv(table_name, csv_path, opts.merge(columns: cols))
     end
 
     def load_data_from_csv(table_name, csv_path, opts={})
       columns = opts[:columns] || get_csv_headers(csv_path)
 
-      # temporary files to get the excepted records
-      exc = opts[:exceptions_file] ||= Tempfile.new('exceptions')
-      rej = opts[:rejections_file] ||= Tempfile.new('rejections')
-      exc = File.new(exc) unless exc.is_a?(File)
-      rej = File.new(rej) unless exc.is_a?(File)
+      if opts[:ignore_parse_errors] && opts[:exceptions_file].nil? && opts[:rejections_file].nil?
+        exc = nil
+        rej = nil
+      else
+        # temporary files to get the excepted records (if not given)
+        exc = opts[:exceptions_file] ||= Tempfile.new('exceptions')
+        rej = opts[:rejections_file] ||= Tempfile.new('rejections')
+        exc = File.new(exc) unless exc.is_a?(File)
+        rej = File.new(rej) unless rej.is_a?(File)
+      end
 
       # execute the load
       execute(GoodData::SQLGenerator.load_data(table_name, csv_path, columns, opts))
 
-      exc.close
-      rej.close
+      exc.close if exc
+      rej.close if rej
 
-      if exc.size > 0 || rej.size > 0
+      # if there was something rejected and it shouldn't be ignored, raise an error
+      if ((exc && File.size?(exc)) || (rej && File.size?(rej))) && (! opts[:ignore_parse_errors])
         fail ArgumentError, "Some lines in the CSV didn't go through. Exceptions: #{IO.read(exc)}\nRejected records: #{IO.read(rej)}"
       end
     end

data/lib/gooddata_datawarehouse/sql_generator.rb

@@ -25,7 +25,11 @@ module GoodData
         parser = opts[:parser] || 'GdcCsvParser()'
         escape_as = opts[:escape_as] || '"'
 
-        exc_rej = opts[:ignore_parse_errors] ? '' : "EXCEPTIONS '#{File.absolute_path(opts[:exceptions_file])}' REJECTED DATA '#{File.absolute_path(opts[:rejections_file])}'"
+        exc_rej = if opts[:ignore_parse_errors] && opts[:exceptions_file].nil? && opts[:rejections_file].nil?
+                    ''
+                  else
+                    "EXCEPTIONS '#{File.absolute_path(opts[:exceptions_file])}' REJECTED DATA '#{File.absolute_path(opts[:rejections_file])}'"
+                  end
 
         %Q{COPY #{table} (#{col_list})
         FROM LOCAL '#{File.absolute_path(csv)}' WITH PARSER #{parser}

data/lib/gooddata_datawarehouse/version.rb

@@ -1,5 +1,5 @@
 module GoodData
   class Datawarehouse
-    VERSION = "0.0.3"
+    VERSION = "0.0.4"
   end
 end

data/spec/datawarehouse_spec.rb

@@ -2,6 +2,9 @@ require 'tempfile'
 require 'gooddata_datawarehouse/datawarehouse'
 require_relative 'spec_helper'
 
+CSV_PATH = 'spec/data/bike.csv'
+WRONG_CSV_PATH = 'spec/data/bike-wrong.csv'
+
 describe GoodData::Datawarehouse do
   before(:each) do
     @dwh = SpecHelper::create_default_connection
@@ -100,22 +103,80 @@ describe GoodData::Datawarehouse do
 
   describe '#csv_to_new_table' do
     it 'creates a new table from csv' do
-      path = 'spec/data/bike.csv'
-      @dwh.csv_to_new_table(@random_table_name, path)
+      @dwh.csv_to_new_table(@random_table_name, CSV_PATH)
+
+      # table exists
+      expect(@dwh.table_exists?(@random_table_name)).to eq true
+
+      # cols are the same as in the csv
+      expected_cols = File.open(CSV_PATH, &:gets).strip.split(',')
+      expect(Set.new(@dwh.get_columns(@random_table_name))).to eq Set.new(expected_cols.map {|c| {:column_name => c, :data_type => GoodData::SQLGenerator::DEFAULT_TYPE}})
+    end
+
+    it 'writes exceptions and rejections to files at given path, passed strings' do
+      rej = Tempfile.new('rejections.csv')
+      exc = Tempfile.new('exceptions.csv')
+
+      @dwh.csv_to_new_table(@random_table_name, CSV_PATH, :exceptions_file => exc.path, :rejections_file => rej.path)
+
+      expect(File.size(rej)).to eq 0
+      expect(File.size(exc)).to eq 0
+    end
+
+    it 'writes exceptions and rejections to files at given path, passed files' do
+      rej = Tempfile.new('rejections.csv')
+      exc = Tempfile.new('exceptions.csv')
+
+      @dwh.csv_to_new_table(@random_table_name, CSV_PATH, :exceptions_file => exc, :rejections_file => rej)
+
+      expect(File.size(rej)).to eq 0
+      expect(File.size(exc)).to eq 0
+    end
+
+    it "writes exceptions and rejections to files at given absolute path, when it's wrong there's something" do
+      rej = Tempfile.new('rejections.csv')
+      exc = Tempfile.new('exceptions.csv')
+
+      @dwh.csv_to_new_table(@random_table_name, WRONG_CSV_PATH, :exceptions_file => exc.path, :rejections_file => rej.path, :ignore_parse_errors => true)
+
+      expect(File.size(rej)).to be > 0
+      expect(File.size(exc)).to be > 0
+    end
+
+    it "writes exceptions and rejections to files at given relative path, when it's wrong there's something" do
+      rej = Tempfile.new('rejections.csv')
+      exc = Tempfile.new('exceptions.csv')
+
+      if File.dirname(rej) != File.dirname(exc)
+        raise "two directories for tempfiles!"
+      end
+
+      csv_path = File.expand_path(WRONG_CSV_PATH)
+
+      Dir.chdir(File.dirname(rej)) do
+        @dwh.csv_to_new_table(@random_table_name, csv_path, :exceptions_file => File.basename(exc), :rejections_file => File.basename(rej), :ignore_parse_errors => true)
+      end
+
+
+      expect(File.size(rej)).to be > 0
+      expect(File.size(exc)).to be > 0
+    end
+
+    it "does something when ignoring errors and not passing files" do
+      @dwh.csv_to_new_table(@random_table_name, CSV_PATH, :ignore_parse_errors => true)
 
       # table exists
       expect(@dwh.table_exists?(@random_table_name)).to eq true
 
       # cols are the same as in the csv
-      expected_cols = File.open(path, &:gets).strip.split(',')
+      expected_cols = File.open(CSV_PATH, &:gets).strip.split(',')
       expect(Set.new(@dwh.get_columns(@random_table_name))).to eq Set.new(expected_cols.map {|c| {:column_name => c, :data_type => GoodData::SQLGenerator::DEFAULT_TYPE}})
     end
   end
 
   describe '#export_table' do
     it 'exports a created table' do
-      path = 'spec/data/bike.csv'
-      @dwh.csv_to_new_table(@random_table_name, path)
+      @dwh.csv_to_new_table(@random_table_name, CSV_PATH)
 
       # table exists
       expect(@dwh.table_exists?(@random_table_name)).to eq true
@@ -125,7 +186,7 @@ describe GoodData::Datawarehouse do
       @dwh.export_table(@random_table_name, f)
 
       # should be the same except for order of the lines
-      imported = Set.new(CSV.read(path))
+      imported = Set.new(CSV.read(CSV_PATH))
       exported = Set.new(CSV.read(f))
 
       expect(exported).to eq imported
@@ -134,39 +195,42 @@ describe GoodData::Datawarehouse do
 
   describe '#load_data_from_csv' do
     it 'loads data from csv to existing table' do
-      path = 'spec/data/bike.csv'
-
       # create the table
-      @dwh.create_table_from_csv_header(@random_table_name, path)
+      @dwh.create_table_from_csv_header(@random_table_name, CSV_PATH)
       expect(@dwh.table_exists?(@random_table_name)).to eq true
 
-      expected_cols = File.open(path, &:gets).strip.split(',')
+      expected_cols = File.open(CSV_PATH, &:gets).strip.split(',')
       expect(Set.new(@dwh.get_columns(@random_table_name))).to eq Set.new(expected_cols.map {|c| {:column_name => c, :data_type => GoodData::SQLGenerator::DEFAULT_TYPE}})
 
       # load the data there
-      @dwh.load_data_from_csv(@random_table_name, path)
+      @dwh.load_data_from_csv(@random_table_name, CSV_PATH)
 
       # export it
       f = Tempfile.new('bike.csv')
       @dwh.export_table(@random_table_name, f)
 
       # should be the same except for order of the lines
-      imported = Set.new(CSV.read(path))
+      imported = Set.new(CSV.read(CSV_PATH))
       exported = Set.new(CSV.read(f))
 
       expect(exported).to eq imported
     end
 
     it 'fails for a wrong csv' do
-      path_wrong = 'spec/data/bike-wrong.csv'
-
       # create the table
-      @dwh.create_table_from_csv_header(@random_table_name, path_wrong)
+      @dwh.create_table_from_csv_header(@random_table_name, WRONG_CSV_PATH)
       expect(@dwh.table_exists?(@random_table_name)).to eq true
 
       # load the data there - expect fail
-      expect{@dwh.load_data_from_csv(@random_table_name, path_wrong)}.to raise_error(ArgumentError)
+      expect{@dwh.load_data_from_csv(@random_table_name, WRONG_CSV_PATH)}.to raise_error(ArgumentError)
+    end
+  end
 
+  describe '#get_columns' do
+    it 'gives you the right list of columns' do
+      expected_cols = File.open(CSV_PATH, &:gets).strip.split(',')
+      @dwh.create_table_from_csv_header(@random_table_name, CSV_PATH)
     end
+    # TODO more tests
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gooddata_datawarehouse
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
 platform: ruby
 authors:
 - Petr Cvengros
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-01-07 00:00:00.000000000 Z
+date: 2015-01-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement