gooddata_datawarehouse 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6a3ba4f26ae8e33a6118a1f0a892315968f92857
-  data.tar.gz: 2be9b35b829f8910116a65252a3651ab6dfd5be6
+  metadata.gz: c6dab944c3225fc70d395c43efb00edf49cd9211
+  data.tar.gz: 43a0ea9ca6328951e8f5087d97ba87014387d2be
 SHA512:
-  metadata.gz: c2b74588cb194e9c0928dcd3c6c3250bf64907d1fe4fc93d137506a2f2f8e86c02350cbdbbc7341c5854a30d75ac7ad36b2269e8f3c5f393959f6b4ee1b26b59
-  data.tar.gz: e7e8b1cb11d8926b47d516e9a636b79096d055bb7ca35331bb6edcf5d67926fde3a57ebb32c719741f83cf0caa2e6da66f2f0688535be91e13158e55af446113
+  metadata.gz: 67768d3deca6993494a62c4de9c05d4f62226ee1806d853d8183f529aa8510780ff8856331cfe6502259b871ebc8b3a26b70a482773901a6e691516836d02be2
+  data.tar.gz: 67bae1529cd48d066790a10e7b4927daf05110c3440cc3c45da2faed208b4fa1276d0eb02bf52763a98f0b41cde8df2400ecade369c3efdc859ae7515e62efde

data/.travis.yml

@@ -0,0 +1,21 @@
+language: ruby
+sudo: false
+os:
+- linux
+- osx
+branches:
+except:
+- gh-pages
+- next_doc_release
+env:
+  global:
+    - secure: "hPFBMIHa0J+Dc1gBPqipkr+4RLFQGQWBTHZbPRoAU/r9n8NkH36KvLjBNINgvJ/7eFNR/BLBFZ5tylp1zTp2E3Ysvv3snGN/qIHmAAa1O9YINbAugSUNka00kClH4s0vWF1r+e4uKTclb2TFhVBSRLjIAVEXibAQuXd7nMCm+78="
+    - secure: "Rqt5jW8L5c+3sVIZcfDRkXjlpy6P4g3F7JGliBcOP1whKOiJW15l5yiANoLolTBxps2UneewxcxViI4oa+ZIUZr6xms0on2I9OTyK69wZ62fVrwwrTyI+FwPfF0ilvieuuU2TF9i3bEYB9XyZHQMgqmnX4DtNFvX6hUnVPQsSn8="
+    - secure: "O1KhJhV69JooBAXjvZmLAbpBWTmwk0wlgQP3vlYshHC4HnGYh4su80dESn5NmTry8QuxYZNDAh211pkVx0/r73y6/NJfDd5eO1KKR+fR0HuLaRkhEW5PjfvZxkwDSa2Dwi1WZDOtd/3+3l26RGl0tg6A08fAUXgM2W7yKkB43L4="
+rvm:
+- jruby-19mode
+before_install:
+- gem update --system
+- gem update bundler
+script: rake ci
+

data/README.md

@@ -2,17 +2,36 @@
 
 A little library to help you work with GoodData's Datawarehouse
 
+## Status
+
+[![Gem Version](https://badge.fury.io/rb/gooddata_datawarehouse.png)](http://badge.fury.io/rb/gooddata_datawarehouse)
+[![Downloads](http://img.shields.io/gem/dt/gooddata_datawarehouse.svg)](http://rubygems.org/gems/gooddata_datawarehouse)
+[![Dependency Status](https://gemnasium.com/cvengros/gooddata_datawarehouse.png)](https://gemnasium.com/cvengros/gooddata_datawarehouse)
+[![Code Climate](https://codeclimate.com/github/cvengros/gooddata_datawarehouse.png)](https://codeclimate.com/github/cvengros/gooddata_datawarehouse)
+[![Build Status](https://travis-ci.org/cvengros/gooddata_datawarehouse.png)](https://travis-ci.org/cvengros/gooddata_datawarehouse)
+[![Coverage Status](https://coveralls.io/repos/cvengros/gooddata_datawarehouse/badge.png)](https://coveralls.io/r/cvengros/gooddata_datawarehouse)
+
 ## Installation
 
+You need to run **jRuby** to use this gem, the gem won't work on any other Ruby platform than jRuby. That's because there's a dependency on the JDBC driver
+
+If you're using [rvm](https://rvm.io/rvm/install) (recommended), run:
+    
+    $ rvm use jruby
+
+If you don't have jruby yet, run
+
+    $ rvm install jruby
+
 Add this line to your application's Gemfile:
 
 ```ruby
 gem 'gooddata_datawarehouse'
 ```
 
-And then execute:
+And then install:
 
-    $ bundle
+    $ bundle install
 
 Or install it yourself as:
 
@@ -24,7 +43,7 @@ Or install it yourself as:
 require 'gooddata_datawarehouse'
 
 # connect
-dwh = GoodData::Datawarehouse.new('you@gooddata.com', 'yourpass', 'your ADS instance id'
+dwh = GoodData::Datawarehouse.new('you@gooddata.com', 'yourpass', 'your ADS instance id')
 
 # import a csv
 dwh.csv_to_new_table('my_table', 'path/to/my.csv')
@@ -40,7 +59,6 @@ dwh.execute_select('SELECT * FROM my_table ORDER BY col1') do |row|
   puts row[:col1] 
 end
 
-
 # rename a table
 dwh.rename_table('my_table', 'my_new_table')
 

data/Rakefile

@@ -1,2 +1,20 @@
-require "bundler/gem_tasks"
+require 'coveralls/rake/task'
+require 'rake/testtask'
+require 'rspec/core/rake_task'
 
+Coveralls::RakeTask.new
+
+task default: %w[ci]
+
+desc 'Run continuous integration test'
+task :ci do
+  Rake::Task['test:unit'].invoke
+  Rake::Task['coveralls:push'].invoke
+end
+
+namespace :test do
+  desc "Run unit tests"
+  RSpec::Core::RakeTask.new(:unit) do |t|
+    t.pattern = 'spec/**/*.rb'
+  end
+end

data/gooddata_datawarehouse.gemspec

@@ -23,9 +23,10 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_development_dependency "bundler", "~> 1.7"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", "~> 10.3"
   spec.add_development_dependency 'rspec', '~>2.14'
   spec.add_development_dependency 'pry', '~> 0.9'
+  spec.add_development_dependency 'coveralls', '~> 0.7', '>= 0.7.0'
 
   spec.add_dependency "sequel", "~> 4.17"
   spec.add_dependency "gooddata-dss-jdbc", "~> 0.1"

data/lib/gooddata_datawarehouse/datawarehouse.rb

@@ -7,11 +7,15 @@ require_relative 'sql_generator'
 
 module GoodData
   class Datawarehouse
-    def initialize(username, password, instance_id, options={})
+    def initialize(username, password, instance_id, opts={})
       @logger = Logger.new(STDOUT)
       @username = username
       @password = password
       @jdbc_url = "jdbc:dss://secure.gooddata.com/gdc/dss/instances/#{instance_id}"
+      if @username.nil? || @password.nil? || instance_id.nil?
+        fail ArgumentError, "username, password and/or instance_id are nil. All of them are mandatory."
+      end
+
       Jdbc::DSS.load_driver
       Java.com.gooddata.dss.jdbc.driver.DssDriver
     end
@@ -20,7 +24,6 @@ module GoodData
       CSV.open(csv_path, 'wb', :force_quotes => true) do |csv|
         # get the names of cols
         cols = get_columns(table_name).map {|c| c[:column_name]}
-        col_names =
 
         # write header
         csv << cols
@@ -53,7 +56,22 @@ module GoodData
 
     def load_data_from_csv(table_name, csv_path, opts={})
       columns = opts[:columns] || get_csv_headers(csv_path)
-      execute(GoodData::SQLGenerator.load_data(table_name, csv_path, columns))
+
+      # temporary files to get the excepted records
+      exc = opts[:exceptions_file] ||= Tempfile.new('exceptions')
+      rej = opts[:rejections_file] ||= Tempfile.new('rejections')
+      exc = File.new(exc) unless exc.is_a?(File)
+      rej = File.new(rej) unless exc.is_a?(File)
+
+      # execute the load
+      execute(GoodData::SQLGenerator.load_data(table_name, csv_path, columns, opts))
+
+      exc.close
+      rej.close
+
+      if exc.size > 0 || rej.size > 0
+        fail ArgumentError, "Some lines in the CSV didn't go through. Exceptions: #{IO.read(exc)}\nRejected records: #{IO.read(rej)}"
+      end
     end
 
     # returns a list of columns created
@@ -65,8 +83,8 @@ module GoodData
       columns
     end
 
-    def create_table(name, columns, options={})
-      execute(GoodData::SQLGenerator.create_table(name, columns, options))
+    def create_table(name, columns, opts={})
+      execute(GoodData::SQLGenerator.create_table(name, columns, opts))
     end
 
     def table_exists?(name)
@@ -92,9 +110,9 @@ module GoodData
     end
 
     # executes sql (select), for each row, passes execution to block
-    def execute_select(sql, options={})
-      fetch_handler = options[:fetch_handler]
-      count = options[:count]
+    def execute_select(sql, opts={})
+      fetch_handler = opts[:fetch_handler]
+      count = opts[:count]
 
       connect do |connection|
         # do the query

data/lib/gooddata_datawarehouse/sql_generator.rb

@@ -19,17 +19,19 @@ module GoodData
         "CREATE TABLE #{not_exists} #{table_name} (#{columns_string})"
       end
 
-      def load_data(table, csv, columns)
+      def load_data(table, csv, columns, opts={})
         col_list = columns.join(', ')
+        skip = opts[:no_header] ? '' : 'SKIP 1'
+        parser = opts[:parser] || 'GdcCsvParser()'
+        escape_as = opts[:escape_as] || '"'
 
-        # TODO: exceptions, rejections
-        #       EXCEPTIONS '#{except_filename(filename)}'
-        # REJECTED DATA '#{reject_filename(filename)}' }
+        exc_rej = opts[:ignore_parse_errors] ? '' : "EXCEPTIONS '#{File.absolute_path(opts[:exceptions_file])}' REJECTED DATA '#{File.absolute_path(opts[:rejections_file])}'"
 
         %Q{COPY #{table} (#{col_list})
-        FROM LOCAL '#{csv}' WITH PARSER GdcCsvParser()
-        ESCAPE AS '"'
-         SKIP 1}
+        FROM LOCAL '#{File.absolute_path(csv)}' WITH PARSER #{parser}
+        ESCAPE AS '#{escape_as}'
+        #{skip}
+        #{exc_rej}}
       end
 
       def get_table_count(table_name)
@@ -40,8 +42,8 @@ module GoodData
         "SELECT column_name, data_type FROM columns WHERE table_name = '#{table_name}'"
       end
 
-      def select_all(table_name, options={})
-        limit = options[:limit] ? "LIMIT #{options[:limit]}" : ''
+      def select_all(table_name, opts={})
+        limit = opts[:limit] ? "LIMIT #{opts[:limit]}" : ''
         "SELECT * FROM #{table_name} #{limit}"
       end
     end

data/lib/gooddata_datawarehouse/version.rb

@@ -1,5 +1,5 @@
 module GoodData
   class Datawarehouse
-    VERSION = "0.0.1"
+    VERSION = "0.0.2"
   end
 end

data/spec/data/bike-wrong.csv

@@ -0,0 +1,6 @@
+wheel_size,manufacturer
+"29","Specialized"
+"27.5","Scott"
+"29","Scott"
+"989","989"
+"26","Something","Too much cols"

data/spec/datawarehouse_spec.rb

@@ -11,14 +11,14 @@ describe GoodData::Datawarehouse do
   end
 
   after(:each) do
-    @created_tables.each{|t| @dwh.drop_table(t)} if @created_tables
+    @created_tables ||= [@random_table_name]
+    @created_tables.each{|t| @dwh.drop_table(t) if t} if @created_tables
   end
 
   describe '#create_table' do
     it 'creates a table with default type' do
       cols = ['col1', 'col2', 'col3']
       @dwh.create_table(@random_table_name, cols)
-      @created_tables = [@random_table_name]
 
       # table exists
       expect(@dwh.table_exists?(@random_table_name)).to eq true
@@ -31,7 +31,6 @@ describe GoodData::Datawarehouse do
       cols = ['col1', 'col2', 'col3']
       cols2 = ['col1', 'col2']
       @dwh.create_table(@random_table_name, cols)
-      @created_tables = [@random_table_name]
 
       expect(@dwh.table_exists?(@random_table_name)).to eq true
 
@@ -58,7 +57,6 @@ describe GoodData::Datawarehouse do
         }
       ]
       @dwh.create_table(@random_table_name, cols)
-      @created_tables = [@random_table_name]
 
       # table exists
       expect(@dwh.table_exists?(@random_table_name)).to eq true
@@ -78,6 +76,8 @@ describe GoodData::Datawarehouse do
       # it shouldn't exist after being dropped
       @dwh.drop_table(@random_table_name)
       expect(@dwh.table_exists?(@random_table_name)).to eq false
+
+      @random_table_name = nil
     end
   end
 
@@ -109,7 +109,6 @@ describe GoodData::Datawarehouse do
       # cols are the same as in the csv
       expected_cols = File.open(path, &:gets).strip.split(',')
       expect(Set.new(@dwh.get_columns(@random_table_name))).to eq Set.new(expected_cols.map {|c| {:column_name => c, :data_type => GoodData::SQLGenerator::DEFAULT_TYPE}})
-      @created_tables = [@random_table_name]
     end
   end
 
@@ -130,7 +129,6 @@ describe GoodData::Datawarehouse do
       exported = Set.new(CSV.read(f))
 
       expect(exported).to eq imported
-      @created_tables = [@random_table_name]
     end
   end
 
@@ -157,7 +155,6 @@ describe GoodData::Datawarehouse do
       exported = Set.new(CSV.read(f))
 
       expect(exported).to eq imported
-      @created_tables = [@random_table_name]
     end
 
     it 'fails for a wrong csv' do
@@ -167,8 +164,8 @@ describe GoodData::Datawarehouse do
       @dwh.create_table_from_csv_header(@random_table_name, path_wrong)
       expect(@dwh.table_exists?(@random_table_name)).to eq true
 
-      # load the data there
-      @dwh.load_data_from_csv(@random_table_name, path_wrong)
+      # load the data there - expect fail
+      expect{@dwh.load_data_from_csv(@random_table_name, path_wrong)}.to raise_error(ArgumentError)
 
     end
   end

data/spec/spec_helper.rb

@@ -1,4 +1,7 @@
 require 'gooddata_datawarehouse'
+require 'coveralls'
+
+Coveralls.wear_merged!
 
 RSpec.configure do |c|
   c.filter_run :focus => true

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: gooddata_datawarehouse
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Petr Cvengros
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2014-12-18 00:00:00.000000000 Z
+date: 2015-01-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -29,7 +29,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '10.3'
   name: rake
   prerelease: false
   type: :development
@@ -37,7 +37,7 @@ dependencies:
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '10.3'
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -66,6 +66,26 @@ dependencies:
     - - ~>
       - !ruby/object:Gem::Version
         version: '0.9'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.7'
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.7.0
+  name: coveralls
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.7'
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 0.7.0
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
     requirements:
@@ -102,6 +122,7 @@ extensions: []
 extra_rdoc_files: []
 files:
 - .gitignore
+- .travis.yml
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -112,6 +133,7 @@ files:
 - lib/gooddata_datawarehouse/datawarehouse.rb
 - lib/gooddata_datawarehouse/sql_generator.rb
 - lib/gooddata_datawarehouse/version.rb
+- spec/data/bike-wrong.csv
 - spec/data/bike.csv
 - spec/datawarehouse_spec.rb
 - spec/spec_helper.rb
@@ -140,6 +162,7 @@ signing_key:
 specification_version: 4
 summary: Convenient work with GoodData's Datawarehouse (ADS)
 test_files:
+- spec/data/bike-wrong.csv
 - spec/data/bike.csv
 - spec/datawarehouse_spec.rb
 - spec/spec_helper.rb