gooddata_datawarehouse 0.0.1 → 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.travis.yml +21 -0
- data/README.md +22 -4
- data/Rakefile +19 -1
- data/gooddata_datawarehouse.gemspec +2 -1
- data/lib/gooddata_datawarehouse/datawarehouse.rb +26 -8
- data/lib/gooddata_datawarehouse/sql_generator.rb +11 -9
- data/lib/gooddata_datawarehouse/version.rb +1 -1
- data/spec/data/bike-wrong.csv +6 -0
- data/spec/datawarehouse_spec.rb +6 -9
- data/spec/spec_helper.rb +3 -0
- metadata +27 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c6dab944c3225fc70d395c43efb00edf49cd9211
|
4
|
+
data.tar.gz: 43a0ea9ca6328951e8f5087d97ba87014387d2be
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 67768d3deca6993494a62c4de9c05d4f62226ee1806d853d8183f529aa8510780ff8856331cfe6502259b871ebc8b3a26b70a482773901a6e691516836d02be2
|
7
|
+
data.tar.gz: 67bae1529cd48d066790a10e7b4927daf05110c3440cc3c45da2faed208b4fa1276d0eb02bf52763a98f0b41cde8df2400ecade369c3efdc859ae7515e62efde
|
data/.travis.yml
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
language: ruby
|
2
|
+
sudo: false
|
3
|
+
os:
|
4
|
+
- linux
|
5
|
+
- osx
|
6
|
+
branches:
|
7
|
+
except:
|
8
|
+
- gh-pages
|
9
|
+
- next_doc_release
|
10
|
+
env:
|
11
|
+
global:
|
12
|
+
- secure: "hPFBMIHa0J+Dc1gBPqipkr+4RLFQGQWBTHZbPRoAU/r9n8NkH36KvLjBNINgvJ/7eFNR/BLBFZ5tylp1zTp2E3Ysvv3snGN/qIHmAAa1O9YINbAugSUNka00kClH4s0vWF1r+e4uKTclb2TFhVBSRLjIAVEXibAQuXd7nMCm+78="
|
13
|
+
- secure: "Rqt5jW8L5c+3sVIZcfDRkXjlpy6P4g3F7JGliBcOP1whKOiJW15l5yiANoLolTBxps2UneewxcxViI4oa+ZIUZr6xms0on2I9OTyK69wZ62fVrwwrTyI+FwPfF0ilvieuuU2TF9i3bEYB9XyZHQMgqmnX4DtNFvX6hUnVPQsSn8="
|
14
|
+
- secure: "O1KhJhV69JooBAXjvZmLAbpBWTmwk0wlgQP3vlYshHC4HnGYh4su80dESn5NmTry8QuxYZNDAh211pkVx0/r73y6/NJfDd5eO1KKR+fR0HuLaRkhEW5PjfvZxkwDSa2Dwi1WZDOtd/3+3l26RGl0tg6A08fAUXgM2W7yKkB43L4="
|
15
|
+
rvm:
|
16
|
+
- jruby-19mode
|
17
|
+
before_install:
|
18
|
+
- gem update --system
|
19
|
+
- gem update bundler
|
20
|
+
script: rake ci
|
21
|
+
|
data/README.md
CHANGED
@@ -2,17 +2,36 @@
|
|
2
2
|
|
3
3
|
A little library to help you work with GoodData's Datawarehouse
|
4
4
|
|
5
|
+
## Status
|
6
|
+
|
7
|
+
[![Gem Version](https://badge.fury.io/rb/gooddata_datawarehouse.png)](http://badge.fury.io/rb/gooddata_datawarehouse)
|
8
|
+
[![Downloads](http://img.shields.io/gem/dt/gooddata_datawarehouse.svg)](http://rubygems.org/gems/gooddata_datawarehouse)
|
9
|
+
[![Dependency Status](https://gemnasium.com/cvengros/gooddata_datawarehouse.png)](https://gemnasium.com/cvengros/gooddata_datawarehouse)
|
10
|
+
[![Code Climate](https://codeclimate.com/github/cvengros/gooddata_datawarehouse.png)](https://codeclimate.com/github/cvengros/gooddata_datawarehouse)
|
11
|
+
[![Build Status](https://travis-ci.org/cvengros/gooddata_datawarehouse.png)](https://travis-ci.org/cvengros/gooddata_datawarehouse)
|
12
|
+
[![Coverage Status](https://coveralls.io/repos/cvengros/gooddata_datawarehouse/badge.png)](https://coveralls.io/r/cvengros/gooddata_datawarehouse)
|
13
|
+
|
5
14
|
## Installation
|
6
15
|
|
16
|
+
You need to run **jRuby** to use this gem, the gem won't work on any other Ruby platform than jRuby. That's because there's a dependency on the JDBC driver
|
17
|
+
|
18
|
+
If you're using [rvm](https://rvm.io/rvm/install) (recommended), run:
|
19
|
+
|
20
|
+
$ rvm use jruby
|
21
|
+
|
22
|
+
If you don't have jruby yet, run
|
23
|
+
|
24
|
+
$ rvm install jruby
|
25
|
+
|
7
26
|
Add this line to your application's Gemfile:
|
8
27
|
|
9
28
|
```ruby
|
10
29
|
gem 'gooddata_datawarehouse'
|
11
30
|
```
|
12
31
|
|
13
|
-
And then
|
32
|
+
And then install:
|
14
33
|
|
15
|
-
$ bundle
|
34
|
+
$ bundle install
|
16
35
|
|
17
36
|
Or install it yourself as:
|
18
37
|
|
@@ -24,7 +43,7 @@ Or install it yourself as:
|
|
24
43
|
require 'gooddata_datawarehouse'
|
25
44
|
|
26
45
|
# connect
|
27
|
-
dwh = GoodData::Datawarehouse.new('you@gooddata.com', 'yourpass', 'your ADS instance id'
|
46
|
+
dwh = GoodData::Datawarehouse.new('you@gooddata.com', 'yourpass', 'your ADS instance id')
|
28
47
|
|
29
48
|
# import a csv
|
30
49
|
dwh.csv_to_new_table('my_table', 'path/to/my.csv')
|
@@ -40,7 +59,6 @@ dwh.execute_select('SELECT * FROM my_table ORDER BY col1') do |row|
|
|
40
59
|
puts row[:col1]
|
41
60
|
end
|
42
61
|
|
43
|
-
|
44
62
|
# rename a table
|
45
63
|
dwh.rename_table('my_table', 'my_new_table')
|
46
64
|
|
data/Rakefile
CHANGED
@@ -1,2 +1,20 @@
|
|
1
|
-
require
|
1
|
+
require 'coveralls/rake/task'
|
2
|
+
require 'rake/testtask'
|
3
|
+
require 'rspec/core/rake_task'
|
2
4
|
|
5
|
+
Coveralls::RakeTask.new
|
6
|
+
|
7
|
+
task default: %w[ci]
|
8
|
+
|
9
|
+
desc 'Run continuous integration test'
|
10
|
+
task :ci do
|
11
|
+
Rake::Task['test:unit'].invoke
|
12
|
+
Rake::Task['coveralls:push'].invoke
|
13
|
+
end
|
14
|
+
|
15
|
+
namespace :test do
|
16
|
+
desc "Run unit tests"
|
17
|
+
RSpec::Core::RakeTask.new(:unit) do |t|
|
18
|
+
t.pattern = 'spec/**/*.rb'
|
19
|
+
end
|
20
|
+
end
|
@@ -23,9 +23,10 @@ Gem::Specification.new do |spec|
|
|
23
23
|
spec.require_paths = ["lib"]
|
24
24
|
|
25
25
|
spec.add_development_dependency "bundler", "~> 1.7"
|
26
|
-
spec.add_development_dependency "rake", "~> 10.
|
26
|
+
spec.add_development_dependency "rake", "~> 10.3"
|
27
27
|
spec.add_development_dependency 'rspec', '~>2.14'
|
28
28
|
spec.add_development_dependency 'pry', '~> 0.9'
|
29
|
+
spec.add_development_dependency 'coveralls', '~> 0.7', '>= 0.7.0'
|
29
30
|
|
30
31
|
spec.add_dependency "sequel", "~> 4.17"
|
31
32
|
spec.add_dependency "gooddata-dss-jdbc", "~> 0.1"
|
@@ -7,11 +7,15 @@ require_relative 'sql_generator'
|
|
7
7
|
|
8
8
|
module GoodData
|
9
9
|
class Datawarehouse
|
10
|
-
def initialize(username, password, instance_id,
|
10
|
+
def initialize(username, password, instance_id, opts={})
|
11
11
|
@logger = Logger.new(STDOUT)
|
12
12
|
@username = username
|
13
13
|
@password = password
|
14
14
|
@jdbc_url = "jdbc:dss://secure.gooddata.com/gdc/dss/instances/#{instance_id}"
|
15
|
+
if @username.nil? || @password.nil? || instance_id.nil?
|
16
|
+
fail ArgumentError, "username, password and/or instance_id are nil. All of them are mandatory."
|
17
|
+
end
|
18
|
+
|
15
19
|
Jdbc::DSS.load_driver
|
16
20
|
Java.com.gooddata.dss.jdbc.driver.DssDriver
|
17
21
|
end
|
@@ -20,7 +24,6 @@ module GoodData
|
|
20
24
|
CSV.open(csv_path, 'wb', :force_quotes => true) do |csv|
|
21
25
|
# get the names of cols
|
22
26
|
cols = get_columns(table_name).map {|c| c[:column_name]}
|
23
|
-
col_names =
|
24
27
|
|
25
28
|
# write header
|
26
29
|
csv << cols
|
@@ -53,7 +56,22 @@ module GoodData
|
|
53
56
|
|
54
57
|
def load_data_from_csv(table_name, csv_path, opts={})
|
55
58
|
columns = opts[:columns] || get_csv_headers(csv_path)
|
56
|
-
|
59
|
+
|
60
|
+
# temporary files to get the excepted records
|
61
|
+
exc = opts[:exceptions_file] ||= Tempfile.new('exceptions')
|
62
|
+
rej = opts[:rejections_file] ||= Tempfile.new('rejections')
|
63
|
+
exc = File.new(exc) unless exc.is_a?(File)
|
64
|
+
rej = File.new(rej) unless exc.is_a?(File)
|
65
|
+
|
66
|
+
# execute the load
|
67
|
+
execute(GoodData::SQLGenerator.load_data(table_name, csv_path, columns, opts))
|
68
|
+
|
69
|
+
exc.close
|
70
|
+
rej.close
|
71
|
+
|
72
|
+
if exc.size > 0 || rej.size > 0
|
73
|
+
fail ArgumentError, "Some lines in the CSV didn't go through. Exceptions: #{IO.read(exc)}\nRejected records: #{IO.read(rej)}"
|
74
|
+
end
|
57
75
|
end
|
58
76
|
|
59
77
|
# returns a list of columns created
|
@@ -65,8 +83,8 @@ module GoodData
|
|
65
83
|
columns
|
66
84
|
end
|
67
85
|
|
68
|
-
def create_table(name, columns,
|
69
|
-
execute(GoodData::SQLGenerator.create_table(name, columns,
|
86
|
+
def create_table(name, columns, opts={})
|
87
|
+
execute(GoodData::SQLGenerator.create_table(name, columns, opts))
|
70
88
|
end
|
71
89
|
|
72
90
|
def table_exists?(name)
|
@@ -92,9 +110,9 @@ module GoodData
|
|
92
110
|
end
|
93
111
|
|
94
112
|
# executes sql (select), for each row, passes execution to block
|
95
|
-
def execute_select(sql,
|
96
|
-
fetch_handler =
|
97
|
-
count =
|
113
|
+
def execute_select(sql, opts={})
|
114
|
+
fetch_handler = opts[:fetch_handler]
|
115
|
+
count = opts[:count]
|
98
116
|
|
99
117
|
connect do |connection|
|
100
118
|
# do the query
|
@@ -19,17 +19,19 @@ module GoodData
|
|
19
19
|
"CREATE TABLE #{not_exists} #{table_name} (#{columns_string})"
|
20
20
|
end
|
21
21
|
|
22
|
-
def load_data(table, csv, columns)
|
22
|
+
def load_data(table, csv, columns, opts={})
|
23
23
|
col_list = columns.join(', ')
|
24
|
+
skip = opts[:no_header] ? '' : 'SKIP 1'
|
25
|
+
parser = opts[:parser] || 'GdcCsvParser()'
|
26
|
+
escape_as = opts[:escape_as] || '"'
|
24
27
|
|
25
|
-
|
26
|
-
# EXCEPTIONS '#{except_filename(filename)}'
|
27
|
-
# REJECTED DATA '#{reject_filename(filename)}' }
|
28
|
+
exc_rej = opts[:ignore_parse_errors] ? '' : "EXCEPTIONS '#{File.absolute_path(opts[:exceptions_file])}' REJECTED DATA '#{File.absolute_path(opts[:rejections_file])}'"
|
28
29
|
|
29
30
|
%Q{COPY #{table} (#{col_list})
|
30
|
-
FROM LOCAL '#{csv}' WITH PARSER
|
31
|
-
ESCAPE AS '
|
32
|
-
|
31
|
+
FROM LOCAL '#{File.absolute_path(csv)}' WITH PARSER #{parser}
|
32
|
+
ESCAPE AS '#{escape_as}'
|
33
|
+
#{skip}
|
34
|
+
#{exc_rej}}
|
33
35
|
end
|
34
36
|
|
35
37
|
def get_table_count(table_name)
|
@@ -40,8 +42,8 @@ module GoodData
|
|
40
42
|
"SELECT column_name, data_type FROM columns WHERE table_name = '#{table_name}'"
|
41
43
|
end
|
42
44
|
|
43
|
-
def select_all(table_name,
|
44
|
-
limit =
|
45
|
+
def select_all(table_name, opts={})
|
46
|
+
limit = opts[:limit] ? "LIMIT #{opts[:limit]}" : ''
|
45
47
|
"SELECT * FROM #{table_name} #{limit}"
|
46
48
|
end
|
47
49
|
end
|
data/spec/datawarehouse_spec.rb
CHANGED
@@ -11,14 +11,14 @@ describe GoodData::Datawarehouse do
|
|
11
11
|
end
|
12
12
|
|
13
13
|
after(:each) do
|
14
|
-
@created_tables
|
14
|
+
@created_tables ||= [@random_table_name]
|
15
|
+
@created_tables.each{|t| @dwh.drop_table(t) if t} if @created_tables
|
15
16
|
end
|
16
17
|
|
17
18
|
describe '#create_table' do
|
18
19
|
it 'creates a table with default type' do
|
19
20
|
cols = ['col1', 'col2', 'col3']
|
20
21
|
@dwh.create_table(@random_table_name, cols)
|
21
|
-
@created_tables = [@random_table_name]
|
22
22
|
|
23
23
|
# table exists
|
24
24
|
expect(@dwh.table_exists?(@random_table_name)).to eq true
|
@@ -31,7 +31,6 @@ describe GoodData::Datawarehouse do
|
|
31
31
|
cols = ['col1', 'col2', 'col3']
|
32
32
|
cols2 = ['col1', 'col2']
|
33
33
|
@dwh.create_table(@random_table_name, cols)
|
34
|
-
@created_tables = [@random_table_name]
|
35
34
|
|
36
35
|
expect(@dwh.table_exists?(@random_table_name)).to eq true
|
37
36
|
|
@@ -58,7 +57,6 @@ describe GoodData::Datawarehouse do
|
|
58
57
|
}
|
59
58
|
]
|
60
59
|
@dwh.create_table(@random_table_name, cols)
|
61
|
-
@created_tables = [@random_table_name]
|
62
60
|
|
63
61
|
# table exists
|
64
62
|
expect(@dwh.table_exists?(@random_table_name)).to eq true
|
@@ -78,6 +76,8 @@ describe GoodData::Datawarehouse do
|
|
78
76
|
# it shouldn't exist after being dropped
|
79
77
|
@dwh.drop_table(@random_table_name)
|
80
78
|
expect(@dwh.table_exists?(@random_table_name)).to eq false
|
79
|
+
|
80
|
+
@random_table_name = nil
|
81
81
|
end
|
82
82
|
end
|
83
83
|
|
@@ -109,7 +109,6 @@ describe GoodData::Datawarehouse do
|
|
109
109
|
# cols are the same as in the csv
|
110
110
|
expected_cols = File.open(path, &:gets).strip.split(',')
|
111
111
|
expect(Set.new(@dwh.get_columns(@random_table_name))).to eq Set.new(expected_cols.map {|c| {:column_name => c, :data_type => GoodData::SQLGenerator::DEFAULT_TYPE}})
|
112
|
-
@created_tables = [@random_table_name]
|
113
112
|
end
|
114
113
|
end
|
115
114
|
|
@@ -130,7 +129,6 @@ describe GoodData::Datawarehouse do
|
|
130
129
|
exported = Set.new(CSV.read(f))
|
131
130
|
|
132
131
|
expect(exported).to eq imported
|
133
|
-
@created_tables = [@random_table_name]
|
134
132
|
end
|
135
133
|
end
|
136
134
|
|
@@ -157,7 +155,6 @@ describe GoodData::Datawarehouse do
|
|
157
155
|
exported = Set.new(CSV.read(f))
|
158
156
|
|
159
157
|
expect(exported).to eq imported
|
160
|
-
@created_tables = [@random_table_name]
|
161
158
|
end
|
162
159
|
|
163
160
|
it 'fails for a wrong csv' do
|
@@ -167,8 +164,8 @@ describe GoodData::Datawarehouse do
|
|
167
164
|
@dwh.create_table_from_csv_header(@random_table_name, path_wrong)
|
168
165
|
expect(@dwh.table_exists?(@random_table_name)).to eq true
|
169
166
|
|
170
|
-
# load the data there
|
171
|
-
@dwh.load_data_from_csv(@random_table_name, path_wrong)
|
167
|
+
# load the data there - expect fail
|
168
|
+
expect{@dwh.load_data_from_csv(@random_table_name, path_wrong)}.to raise_error(ArgumentError)
|
172
169
|
|
173
170
|
end
|
174
171
|
end
|
data/spec/spec_helper.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: gooddata_datawarehouse
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Petr Cvengros
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2015-01-07 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
@@ -29,7 +29,7 @@ dependencies:
|
|
29
29
|
requirements:
|
30
30
|
- - ~>
|
31
31
|
- !ruby/object:Gem::Version
|
32
|
-
version: '10.
|
32
|
+
version: '10.3'
|
33
33
|
name: rake
|
34
34
|
prerelease: false
|
35
35
|
type: :development
|
@@ -37,7 +37,7 @@ dependencies:
|
|
37
37
|
requirements:
|
38
38
|
- - ~>
|
39
39
|
- !ruby/object:Gem::Version
|
40
|
-
version: '10.
|
40
|
+
version: '10.3'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
requirement: !ruby/object:Gem::Requirement
|
43
43
|
requirements:
|
@@ -66,6 +66,26 @@ dependencies:
|
|
66
66
|
- - ~>
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0.9'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
requirement: !ruby/object:Gem::Requirement
|
71
|
+
requirements:
|
72
|
+
- - ~>
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: '0.7'
|
75
|
+
- - '>='
|
76
|
+
- !ruby/object:Gem::Version
|
77
|
+
version: 0.7.0
|
78
|
+
name: coveralls
|
79
|
+
prerelease: false
|
80
|
+
type: :development
|
81
|
+
version_requirements: !ruby/object:Gem::Requirement
|
82
|
+
requirements:
|
83
|
+
- - ~>
|
84
|
+
- !ruby/object:Gem::Version
|
85
|
+
version: '0.7'
|
86
|
+
- - '>='
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: 0.7.0
|
69
89
|
- !ruby/object:Gem::Dependency
|
70
90
|
requirement: !ruby/object:Gem::Requirement
|
71
91
|
requirements:
|
@@ -102,6 +122,7 @@ extensions: []
|
|
102
122
|
extra_rdoc_files: []
|
103
123
|
files:
|
104
124
|
- .gitignore
|
125
|
+
- .travis.yml
|
105
126
|
- Gemfile
|
106
127
|
- LICENSE.txt
|
107
128
|
- README.md
|
@@ -112,6 +133,7 @@ files:
|
|
112
133
|
- lib/gooddata_datawarehouse/datawarehouse.rb
|
113
134
|
- lib/gooddata_datawarehouse/sql_generator.rb
|
114
135
|
- lib/gooddata_datawarehouse/version.rb
|
136
|
+
- spec/data/bike-wrong.csv
|
115
137
|
- spec/data/bike.csv
|
116
138
|
- spec/datawarehouse_spec.rb
|
117
139
|
- spec/spec_helper.rb
|
@@ -140,6 +162,7 @@ signing_key:
|
|
140
162
|
specification_version: 4
|
141
163
|
summary: Convenient work with GoodData's Datawarehouse (ADS)
|
142
164
|
test_files:
|
165
|
+
- spec/data/bike-wrong.csv
|
143
166
|
- spec/data/bike.csv
|
144
167
|
- spec/datawarehouse_spec.rb
|
145
168
|
- spec/spec_helper.rb
|