gooddata_datawarehouse 0.0.5 → 0.0.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 7f98f560684ad94b0c65263eb0b8bb39748d3fe8
-  data.tar.gz: 5626327514feed7e85f5541871100b3e28373c0d
+  metadata.gz: 9a6a8637a21756e98d892ca84f9abbf1d4cb912e
+  data.tar.gz: a3853b73306ee86d97471d8bec24efa0482c1b26
 SHA512:
-  metadata.gz: 52e171a5c74e80cc056020263d98fe3ca25a28952d7a3ac338356de34003944279ca427370002589d9fb2ad21bdbe32679732f28b19ca2e1c66482e6996fd175
-  data.tar.gz: f285e427b6f9216cd578a9fa9f8ebf32d4a2769a62ed4bb1b03c4ea452009d36b91320463519524e9d97a1aece663155b4f45e731bd84af5f29c4893b41d1f0a
+  metadata.gz: c5b61278e1b619ad6a31ffe9fd947b734aadc61a27090479f51870608385ec07c9cd07a7d35a68b7bbc25b0d60725d0d83e8022e1135ff5e33f773a44062f934
+  data.tar.gz: 3976202ed90034b82109b9ca0e82d1c1fbd6074b7e8aa770ed447a9f1612113205ff2ecc3d2bc55f0c18ca5c6a72be7723237e916bce00c10aa1112e9b4c9429

data/.coveralls.yml

@@ -0,0 +1 @@
+service_name: travis-ci

data/.rspec

@@ -0,0 +1,3 @@
+--color
+--format documentation
+--require spec_helper

data/README.md

@@ -44,6 +44,9 @@ require 'gooddata_datawarehouse'
 
 # connect
 dwh = GoodData::Datawarehouse.new('you@gooddata.com', 'yourpass', 'your ADS instance id')
+# instance id is the identifier of your datawarehouse (ADS). 
+# E.g. for datawarehouse https://secure.gooddata.com/gdc/datawarehouse/instances/d4979ac54df8afb7b5192b0086de6270
+# the instance id is d4979ac54df8afb7b5192b0086de6270
 
 # import a csv
 dwh.csv_to_new_table('my_table', 'path/to/my.csv')
@@ -51,7 +54,7 @@ dwh.csv_to_new_table('my_table', 'path/to/my.csv')
 dwh.table_exists?('my_table') # true
 dwh.get_columns('my_table') # [{column_name: 'col1', data_type: 'varchar(88)'}, {column_name: 'col2', data_type: 'int'}]
 
-# run an aribrary sql
+# run an arbitrary sql
 dwh.execute('ALTER TABLE my_table ADD COLUMN col3 INTEGER')
 
 # run a select and process results 
@@ -69,6 +72,17 @@ dwh.export_table('my_new_table', 'path/to/my_new.csv')
 dwh.drop_table('my_new_table')
 ```
 
+## Troubleshooting
+### Wrong driver version
+If you get an error talking about handshake error and wrong DSS driver version, update your `gooddata-dss-jdbc` gem by running
+
+    $ bundle update
+
+or 
+
+    $ gem update gooddata-dss-jdbc
+
+You should always have the latest version of this gem.
 
 ## Contributing
 

data/gooddata_datawarehouse.gemspec

@@ -30,4 +30,5 @@ Gem::Specification.new do |spec|
 
   spec.add_dependency "sequel", "~> 4.17"
   spec.add_dependency "gooddata-dss-jdbc", "~> 0.1"
+  spec.add_dependency "pmap", "~> 1.0"
 end

data/lib/gooddata_datawarehouse/datawarehouse.rb

@@ -3,11 +3,13 @@ require 'sequel'
 require 'logger'
 require 'csv'
 require 'tempfile'
+require 'pmap'
 
 require_relative 'sql_generator'
 
 module GoodData
   class Datawarehouse
+    PARALEL_COPY_THREAD_COUNT = 5
     def initialize(username, password, instance_id, opts={})
       @logger = Logger.new(STDOUT)
       @username = username
@@ -50,37 +52,57 @@ module GoodData
       execute(GoodData::SQLGenerator.drop_table(table_name,opts))
     end
 
-    def csv_to_new_table(table_name, csv_path, opts={})
-      cols = create_table_from_csv_header(table_name, csv_path, opts)
-      load_data_from_csv(table_name, csv_path, opts.merge(columns: cols))
+    def csv_to_new_table(table_name, csvs, opts={})
+      csv_list = list_files(csvs)
+      cols = create_table_from_csv_header(table_name, csv_list[0], opts)
+      load_data_from_csv(table_name, csv_list, opts.merge(columns: cols))
     end
 
-    def load_data_from_csv(table_name, csv_path, opts={})
-      columns = opts[:columns] || get_csv_headers(csv_path)
-
-      if opts[:ignore_parse_errors] && opts[:exceptions_file].nil? && opts[:rejections_file].nil?
-        exc = nil
-        rej = nil
-      else
-        # temporary files to get the excepted records (if not given)
-        exc = opts[:exceptions_file] ||= Tempfile.new('exceptions')
-        rej = opts[:rejections_file] ||= Tempfile.new('rejections')
-        exc = File.new(exc, 'w') unless exc.is_a?(File)
-        rej = File.new(rej, 'w') unless rej.is_a?(File)
-      end
+    def load_data_from_csv(table_name, csvs, opts={})
+      # get the list of files to load and columns in the csv
+      csv_list = list_files(csvs)
+      columns = opts[:columns] || get_csv_headers(csv_list[0])
+
+      # load each csv from the list
+      single_file = (csv_list.size == 1)
+
+      csv_list.peach(PARALEL_COPY_THREAD_COUNT) do |csv_path|
+        if opts[:ignore_parse_errors] && opts[:exceptions_file].nil? && opts[:rejections_file].nil?
+          exc = nil
+          rej = nil
+          opts_file = opts
+        else
+          opts_file = opts.clone
+          # priradit do opts i do exc -
+          # temporary files to get the excepted records (if not given)
+          exc = opts_file[:exceptions_file] = init_file(opts_file[:exceptions_file], 'exceptions', csv_path, single_file)
+          rej = opts_file[:rejections_file] = init_file(opts_file[:rejections_file], 'rejections', csv_path, single_file)
+        end
 
-      # execute the load
-      execute(GoodData::SQLGenerator.load_data(table_name, csv_path, columns, opts))
+        # execute the load
+        execute(GoodData::SQLGenerator.load_data(table_name, csv_path, columns, opts_file))
 
-      exc.close if exc
-      rej.close if rej
+        exc.close if exc
+        rej.close if rej
 
-      # if there was something rejected and it shouldn't be ignored, raise an error
-      if ((exc && File.size?(exc)) || (rej && File.size?(rej))) && (! opts[:ignore_parse_errors])
-        fail ArgumentError, "Some lines in the CSV didn't go through. Exceptions: #{IO.read(exc)}\nRejected records: #{IO.read(rej)}"
+        # if there was something rejected and it shouldn't be ignored, raise an error
+        if ((exc && File.size?(exc)) || (rej && File.size?(rej))) && (! opts[:ignore_parse_errors])
+          fail ArgumentError, "Some lines in the CSV didn't go through. Exceptions: #{IO.read(exc)}\nRejected records: #{IO.read(rej)}"
+        end
       end
     end
 
+    def init_file(given_filename, key, csv_path, single_file)
+      # only use file postfix if there are multiple files
+      postfix = single_file ? '' : "-#{File.basename(csv_path)}"
+
+      # take what we have and put the source csv name at the end
+      given_filename = given_filename.path if given_filename.is_a?(File)
+      f = "#{given_filename || Tempfile.new(key).path}#{postfix}"
+      f = File.new(f, 'w') unless f.is_a?(File)
+      f
+    end
+
     # returns a list of columns created
     # does nothing if file empty, returns []
     def create_table_from_csv_header(table_name, csv_path, opts={})
@@ -99,6 +121,10 @@ module GoodData
       count > 0
     end
 
+    def table_row_count(table_name)
+      execute_select(GoodData::SQLGenerator.get_row_count(table_name), :count => true)
+    end
+
     def get_columns(table_name)
       res = execute_select(GoodData::SQLGenerator.get_columns(table_name))
     end
@@ -158,6 +184,26 @@ module GoodData
 
     private
 
+    # returns an array of file paths (strings)
+    def list_files(csvs)
+      # csvs can be:
+      case csvs
+        when String
+
+          # directory
+          if File.directory?(csvs)
+            return (Dir.entries(csvs) - ['.', '..']).map{|f| File.join(csvs, f)}
+          end
+
+          # filename or pattern
+          return Dir.glob(csvs)
+
+        # array
+        when Array
+          return csvs
+      end
+    end
+
     def get_csv_headers(csv_path)
       header_str = File.open(csv_path, &:gets)
       if header_str.nil? || header_str.empty?

data/lib/gooddata_datawarehouse/sql_generator.rb

@@ -42,6 +42,10 @@ module GoodData
         "SELECT COUNT(*) FROM tables WHERE table_name = '#{table_name}'"
       end
 
+      def get_row_count(table_name)
+        "SELECT COUNT(*) FROM #{table_name}"
+      end
+
       def get_columns(table_name)
         "SELECT column_name, data_type FROM columns WHERE table_name = '#{table_name}'"
       end

data/lib/gooddata_datawarehouse/version.rb

@@ -1,5 +1,5 @@
 module GoodData
   class Datawarehouse
-    VERSION = "0.0.5"
+    VERSION = "0.0.6"
   end
 end

data/spec/data/bike2.csv

@@ -0,0 +1,4 @@
+wheel_size,manufacturer
+"29","Ibis"
+"27.5","Seven"
+"29","Moots"

data/spec/datawarehouse_spec.rb

@@ -1,13 +1,25 @@
 require 'tempfile'
 require 'gooddata_datawarehouse/datawarehouse'
-require_relative 'spec_helper'
 
 CSV_PATH = 'spec/data/bike.csv'
-WRONG_CSV_PATH = 'spec/data/bike-wrong.csv'
+CSV_PATH2 = 'spec/data/bike2.csv'
+WRONG_CSV_PATH = 'spec/data/wrong-bike.csv'
+CSV_REGEXP = 'spec/data/bike*.csv'
+
+class Helper
+  def self.create_default_connection
+    GoodData::Datawarehouse.new(ENV['USERNAME'], ENV['PASSWORD'], ENV['INSTANCE_ID'])
+  end
+  def self.line_count(f)
+    i = 0
+    CSV.foreach(f, :headers => true) {|_| i += 1}
+    i
+  end
+end
 
 describe GoodData::Datawarehouse do
   before(:each) do
-    @dwh = SpecHelper::create_default_connection
+    @dwh = Helper::create_default_connection
     @random = rand(10000000).to_s
     @random_table_name = "temp_#{@random}"
     @created_tables = nil
@@ -84,6 +96,22 @@ describe GoodData::Datawarehouse do
     end
   end
 
+  def check_cols
+    # cols are the same as in the csv
+    expected_cols = File.open(CSV_PATH, &:gets).strip.split(',')
+    expect(Set.new(@dwh.get_columns(@random_table_name))).to eq Set.new(expected_cols.map {|c| {:column_name => c, :data_type => GoodData::SQLGenerator::DEFAULT_TYPE}})
+  end
+
+  def check_table_exists
+    # table exists
+    expect(@dwh.table_exists?(@random_table_name)).to eq true
+  end
+
+  def check_row_count
+    # there are lines from both of the csvs
+    expect(@dwh.table_row_count(@random_table_name)).to eq Helper.line_count(CSV_PATH) + Helper.line_count(CSV_PATH2)
+  end
+
   describe '#rename_table' do
     it 'renames a table' do
       cols = ['col1', 'col2', 'col3']
@@ -109,8 +137,38 @@ describe GoodData::Datawarehouse do
       expect(@dwh.table_exists?(@random_table_name)).to eq true
 
       # cols are the same as in the csv
-      expected_cols = File.open(CSV_PATH, &:gets).strip.split(',')
-      expect(Set.new(@dwh.get_columns(@random_table_name))).to eq Set.new(expected_cols.map {|c| {:column_name => c, :data_type => GoodData::SQLGenerator::DEFAULT_TYPE}})
+      check_cols
+    end
+
+
+    it "loads all files in a directory" do
+      # make a tempdir and copy the csvs there
+      Dir.mktmpdir('foo') do |dir|
+        FileUtils.cp(CSV_PATH, dir)
+        FileUtils.cp(CSV_PATH2, dir)
+
+        @dwh.csv_to_new_table(@random_table_name, dir)
+      end
+
+      check_table_exists
+      check_cols
+      check_row_count
+    end
+
+    it "loads all files given in a list" do
+      @dwh.csv_to_new_table(@random_table_name, [CSV_PATH, CSV_PATH2])
+
+      check_table_exists
+      check_cols
+      check_row_count
+    end
+
+    it "loads all files given by a regexp" do
+      @dwh.csv_to_new_table(@random_table_name, CSV_REGEXP)
+
+      check_table_exists
+      check_cols
+      check_row_count
     end
 
     it 'writes exceptions and rejections to files at given path, passed strings' do
@@ -123,6 +181,25 @@ describe GoodData::Datawarehouse do
       expect(File.size(exc)).to eq 0
     end
 
+    it 'overwrites the rejections and exceptions' do
+      rej = Tempfile.new('rejections.csv')
+      exc = Tempfile.new('exceptions.csv')
+
+      @dwh.csv_to_new_table(@random_table_name, WRONG_CSV_PATH, :exceptions_file => exc.path, :rejections_file => rej.path, :ignore_parse_errors => true)
+
+      rej_size = File.size(rej)
+      exc_size = File.size(exc)
+
+      expect(rej_size).to be > 0
+      expect(exc_size).to be > 0
+
+      # load it again and see if it was overwritten - has the same size
+      @dwh.load_data_from_csv(@random_table_name, WRONG_CSV_PATH, :exceptions_file => exc.path, :rejections_file => rej.path, :ignore_parse_errors => true)
+
+      expect(File.size(rej)).to eq rej_size
+      expect(File.size(exc)).to be exc_size
+    end
+
     it 'writes exceptions and rejections to files at given path, passed files' do
       rej = Tempfile.new('rejections.csv')
       exc = Tempfile.new('exceptions.csv')
@@ -162,7 +239,7 @@ describe GoodData::Datawarehouse do
       expect(File.size(exc)).to be > 0
     end
 
-    it "does something when ignoring errors and not passing files" do
+    it "loads fine when ignoring errors and not passing files" do
       @dwh.csv_to_new_table(@random_table_name, CSV_PATH, :ignore_parse_errors => true)
 
       # table exists
@@ -173,7 +250,7 @@ describe GoodData::Datawarehouse do
       expect(Set.new(@dwh.get_columns(@random_table_name))).to eq Set.new(expected_cols.map {|c| {:column_name => c, :data_type => GoodData::SQLGenerator::DEFAULT_TYPE}})
     end
 
-    it "works with non-existing files" do
+    it "works with non-existing rejection/exception files" do
       t = Tempfile.new('haha')
       d = File.dirname(t)
 
@@ -188,6 +265,20 @@ describe GoodData::Datawarehouse do
       expect(File.size(rej)).to be > 0
       expect(File.size(exc)).to be > 0
     end
+
+    it "fails if one of the files is wrong" do
+      expect{@dwh.csv_to_new_table(@random_table_name, [CSV_PATH, WRONG_CSV_PATH])}.to raise_error(ArgumentError)
+    end
+
+    it "creates exceptions / rejections for each file when wanted" do
+      rej = Tempfile.new('rejections.csv')
+      exc = Tempfile.new('exceptions.csv')
+
+      @dwh.csv_to_new_table(@random_table_name, [CSV_PATH, WRONG_CSV_PATH], :exceptions_file => exc.path, :rejections_file => rej.path, :ignore_parse_errors => true)
+
+      expect(File.size("#{rej.path}-#{File.basename(WRONG_CSV_PATH)}")).to be > 0
+      expect(File.size("#{exc.path}-#{File.basename(WRONG_CSV_PATH)}")).to be > 0
+    end
   end
 
   describe '#export_table' do
@@ -232,6 +323,18 @@ describe GoodData::Datawarehouse do
       expect(exported).to eq imported
     end
 
+    it "can load multiple files" do
+      # create the table
+      @dwh.create_table_from_csv_header(@random_table_name, CSV_PATH)
+      check_table_exists
+      check_cols
+
+      # load the data there
+      @dwh.load_data_from_csv(@random_table_name, [CSV_PATH, CSV_PATH2])
+
+      check_row_count
+    end
+
     it 'fails for a wrong csv' do
       # create the table
       @dwh.create_table_from_csv_header(@random_table_name, WRONG_CSV_PATH)

data/spec/spec_helper.rb

@@ -1,15 +1,8 @@
-require 'gooddata_datawarehouse'
 require 'coveralls'
-
-Coveralls.wear_merged!
+Coveralls.wear!
 
 RSpec.configure do |c|
   c.filter_run :focus => true
   c.run_all_when_everything_filtered = true
 end
 
-class SpecHelper
-  def self.create_default_connection
-    GoodData::Datawarehouse.new(ENV['USERNAME'], ENV['PASSWORD'], ENV['INSTANCE_ID'])
-  end
-end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gooddata_datawarehouse
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
 platform: ruby
 authors:
 - Petr Cvengros
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2015-01-15 00:00:00.000000000 Z
+date: 2015-02-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -114,6 +114,20 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.1'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  name: pmap
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
 description: ''
 email:
 - petr.cvengros@gooddata.com
@@ -121,7 +135,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- .coveralls.yml
 - .gitignore
+- .rspec
 - .travis.yml
 - Gemfile
 - LICENSE.txt
@@ -133,8 +149,9 @@ files:
 - lib/gooddata_datawarehouse/datawarehouse.rb
 - lib/gooddata_datawarehouse/sql_generator.rb
 - lib/gooddata_datawarehouse/version.rb
-- spec/data/bike-wrong.csv
 - spec/data/bike.csv
+- spec/data/bike2.csv
+- spec/data/wrong-bike.csv
 - spec/datawarehouse_spec.rb
 - spec/spec_helper.rb
 homepage: ''
@@ -162,7 +179,8 @@ signing_key:
 specification_version: 4
 summary: Convenient work with GoodData's Datawarehouse (ADS)
 test_files:
-- spec/data/bike-wrong.csv
 - spec/data/bike.csv
+- spec/data/bike2.csv
+- spec/data/wrong-bike.csv
 - spec/datawarehouse_spec.rb
 - spec/spec_helper.rb