gooddata_datawarehouse 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +36 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +61 -0
- data/Rakefile +2 -0
- data/env_setup-example.sh +5 -0
- data/gooddata_datawarehouse.gemspec +32 -0
- data/lib/gooddata_datawarehouse/datawarehouse.rb +144 -0
- data/lib/gooddata_datawarehouse/sql_generator.rb +49 -0
- data/lib/gooddata_datawarehouse/version.rb +5 -0
- data/lib/gooddata_datawarehouse.rb +2 -0
- data/spec/data/bike.csv +5 -0
- data/spec/datawarehouse_spec.rb +175 -0
- data/spec/spec_helper.rb +12 -0
- metadata +145 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 6a3ba4f26ae8e33a6118a1f0a892315968f92857
|
4
|
+
data.tar.gz: 2be9b35b829f8910116a65252a3651ab6dfd5be6
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: c2b74588cb194e9c0928dcd3c6c3250bf64907d1fe4fc93d137506a2f2f8e86c02350cbdbbc7341c5854a30d75ac7ad36b2269e8f3c5f393959f6b4ee1b26b59
|
7
|
+
data.tar.gz: e7e8b1cb11d8926b47d516e9a636b79096d055bb7ca35331bb6edcf5d67926fde3a57ebb32c719741f83cf0caa2e6da66f2f0688535be91e13158e55af446113
|
data/.gitignore
ADDED
@@ -0,0 +1,36 @@
|
|
1
|
+
*.gem
|
2
|
+
*.rbc
|
3
|
+
/.config
|
4
|
+
/coverage/
|
5
|
+
/InstalledFiles
|
6
|
+
/pkg/
|
7
|
+
/spec/reports/
|
8
|
+
/test/tmp/
|
9
|
+
/test/version_tmp/
|
10
|
+
/tmp/
|
11
|
+
|
12
|
+
## Specific to RubyMotion:
|
13
|
+
.dat*
|
14
|
+
.repl_history
|
15
|
+
build/
|
16
|
+
|
17
|
+
## Documentation cache and generated files:
|
18
|
+
/.yardoc/
|
19
|
+
/_yardoc/
|
20
|
+
/doc/
|
21
|
+
/rdoc/
|
22
|
+
|
23
|
+
## Environment normalisation:
|
24
|
+
/.bundle/
|
25
|
+
/lib/bundler/man/
|
26
|
+
|
27
|
+
# for a library or gem, you might want to ignore these files since the code is
|
28
|
+
# intended to run in multiple environments; otherwise, check them in:
|
29
|
+
Gemfile.lock
|
30
|
+
.ruby-version
|
31
|
+
.ruby-gemset
|
32
|
+
|
33
|
+
# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
|
34
|
+
.rvmrc
|
35
|
+
|
36
|
+
env_setup.sh
|
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
BSD License
|
2
|
+
Copyright (c) 2013, GoodData Corporation
|
3
|
+
Redistribution and use in source and binary forms, with or without
|
4
|
+
modification, are permitted provided that the following conditions are met:
|
5
|
+
* Redistributions of source code must retain the above copyright notice,
|
6
|
+
this list of conditions and the following disclaimer.
|
7
|
+
* Redistributions in binary form must reproduce the above copyright notice,
|
8
|
+
this list of conditions and the following disclaimer in the documentation
|
9
|
+
and/or other materials provided with the distribution.
|
10
|
+
* Neither the name of GoodData Corporation nor the names of its contributors
|
11
|
+
may be used to endorse or promote products derived from this software
|
12
|
+
without specific prior written permission.
|
13
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
14
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
|
15
|
+
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
16
|
+
ARE DISCLAIMED. IN NO EVENT SHALL GOODDATA CORPORATION BE LIABLE FOR ANY
|
17
|
+
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
18
|
+
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
19
|
+
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
20
|
+
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
21
|
+
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
|
22
|
+
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
data/README.md
ADDED
@@ -0,0 +1,61 @@
|
|
1
|
+
# GooddataDatawarehouse
|
2
|
+
|
3
|
+
A little library to help you work with GoodData's Datawarehouse
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
```ruby
|
10
|
+
gem 'gooddata_datawarehouse'
|
11
|
+
```
|
12
|
+
|
13
|
+
And then execute:
|
14
|
+
|
15
|
+
$ bundle
|
16
|
+
|
17
|
+
Or install it yourself as:
|
18
|
+
|
19
|
+
$ gem install gooddata_datawarehouse
|
20
|
+
|
21
|
+
## Usage
|
22
|
+
|
23
|
+
```ruby
|
24
|
+
require 'gooddata_datawarehouse'
|
25
|
+
|
26
|
+
# connect
|
27
|
+
dwh = GoodData::Datawarehouse.new('you@gooddata.com', 'yourpass', 'your ADS instance id'
|
28
|
+
|
29
|
+
# import a csv
|
30
|
+
dwh.csv_to_new_table('my_table', 'path/to/my.csv')
|
31
|
+
|
32
|
+
dwh.table_exists?('my_table') # true
|
33
|
+
dwh.get_columns('my_table') # [{column_name: 'col1', data_type: 'varchar(88)'}, {column_name: 'col2', data_type: 'int'}]
|
34
|
+
|
35
|
+
# run an aribrary sql
|
36
|
+
dwh.execute('ALTER TABLE my_table ADD COLUMN col3 INTEGER')
|
37
|
+
|
38
|
+
# run a select and process results
|
39
|
+
dwh.execute_select('SELECT * FROM my_table ORDER BY col1') do |row|
|
40
|
+
puts row[:col1]
|
41
|
+
end
|
42
|
+
|
43
|
+
|
44
|
+
# rename a table
|
45
|
+
dwh.rename_table('my_table', 'my_new_table')
|
46
|
+
|
47
|
+
# export to csv
|
48
|
+
dwh.export_table('my_new_table', 'path/to/my_new.csv')
|
49
|
+
|
50
|
+
# drop table
|
51
|
+
dwh.drop_table('my_new_table')
|
52
|
+
```
|
53
|
+
|
54
|
+
|
55
|
+
## Contributing
|
56
|
+
|
57
|
+
1. Fork it ( https://github.com/[my-github-username]/gooddata_datawarehouse/fork )
|
58
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
59
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
60
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
61
|
+
5. Create a new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1,32 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'gooddata_datawarehouse/version'
|
5
|
+
|
6
|
+
if RUBY_PLATFORM != 'java'
|
7
|
+
fail "Only java platform supported! Use jRuby, e.g. rvm use jruby"
|
8
|
+
end
|
9
|
+
|
10
|
+
Gem::Specification.new do |spec|
|
11
|
+
spec.name = "gooddata_datawarehouse"
|
12
|
+
spec.version = GoodData::Datawarehouse::VERSION
|
13
|
+
spec.authors = ["Petr Cvengros"]
|
14
|
+
spec.email = ["petr.cvengros@gooddata.com"]
|
15
|
+
spec.summary = %q{Convenient work with GoodData's Datawarehouse (ADS) }
|
16
|
+
spec.description = ""
|
17
|
+
spec.homepage = ""
|
18
|
+
spec.license = "MIT"
|
19
|
+
|
20
|
+
spec.files = `git ls-files -z`.split("\x0")
|
21
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
22
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
23
|
+
spec.require_paths = ["lib"]
|
24
|
+
|
25
|
+
spec.add_development_dependency "bundler", "~> 1.7"
|
26
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
27
|
+
spec.add_development_dependency 'rspec', '~>2.14'
|
28
|
+
spec.add_development_dependency 'pry', '~> 0.9'
|
29
|
+
|
30
|
+
spec.add_dependency "sequel", "~> 4.17"
|
31
|
+
spec.add_dependency "gooddata-dss-jdbc", "~> 0.1"
|
32
|
+
end
|
@@ -0,0 +1,144 @@
|
|
1
|
+
require 'jdbc/dss'
|
2
|
+
require 'sequel'
|
3
|
+
require 'logger'
|
4
|
+
require 'csv'
|
5
|
+
|
6
|
+
require_relative 'sql_generator'
|
7
|
+
|
8
|
+
module GoodData
|
9
|
+
class Datawarehouse
|
10
|
+
def initialize(username, password, instance_id, options={})
|
11
|
+
@logger = Logger.new(STDOUT)
|
12
|
+
@username = username
|
13
|
+
@password = password
|
14
|
+
@jdbc_url = "jdbc:dss://secure.gooddata.com/gdc/dss/instances/#{instance_id}"
|
15
|
+
Jdbc::DSS.load_driver
|
16
|
+
Java.com.gooddata.dss.jdbc.driver.DssDriver
|
17
|
+
end
|
18
|
+
|
19
|
+
def export_table(table_name, csv_path)
|
20
|
+
CSV.open(csv_path, 'wb', :force_quotes => true) do |csv|
|
21
|
+
# get the names of cols
|
22
|
+
cols = get_columns(table_name).map {|c| c[:column_name]}
|
23
|
+
col_names =
|
24
|
+
|
25
|
+
# write header
|
26
|
+
csv << cols
|
27
|
+
|
28
|
+
# get the keys for columns, stupid sequel
|
29
|
+
col_keys = nil
|
30
|
+
rows = execute_select(GoodData::SQLGenerator.select_all(table_name, limit: 1))
|
31
|
+
|
32
|
+
col_keys = rows[0].keys
|
33
|
+
|
34
|
+
execute_select(GoodData::SQLGenerator.select_all(table_name)) do |row|
|
35
|
+
# go through the table write to csv
|
36
|
+
csv << row.values_at(*col_keys)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def rename_table(old_name, new_name)
|
42
|
+
execute(GoodData::SQLGenerator.rename_table(old_name, new_name))
|
43
|
+
end
|
44
|
+
|
45
|
+
def drop_table(table_name, opts={})
|
46
|
+
execute(GoodData::SQLGenerator.drop_table(table_name,opts))
|
47
|
+
end
|
48
|
+
|
49
|
+
def csv_to_new_table(table_name, csv_path, opts={})
|
50
|
+
cols = create_table_from_csv_header(table_name, csv_path, opts)
|
51
|
+
load_data_from_csv(table_name, csv_path, columns: cols)
|
52
|
+
end
|
53
|
+
|
54
|
+
def load_data_from_csv(table_name, csv_path, opts={})
|
55
|
+
columns = opts[:columns] || get_csv_headers(csv_path)
|
56
|
+
execute(GoodData::SQLGenerator.load_data(table_name, csv_path, columns))
|
57
|
+
end
|
58
|
+
|
59
|
+
# returns a list of columns created
|
60
|
+
# does nothing if file empty, returns []
|
61
|
+
def create_table_from_csv_header(table_name, csv_path, opts={})
|
62
|
+
# take the header as a list of columns
|
63
|
+
columns = get_csv_headers(csv_path)
|
64
|
+
create_table(table_name, columns, opts) unless columns.empty?
|
65
|
+
columns
|
66
|
+
end
|
67
|
+
|
68
|
+
def create_table(name, columns, options={})
|
69
|
+
execute(GoodData::SQLGenerator.create_table(name, columns, options))
|
70
|
+
end
|
71
|
+
|
72
|
+
def table_exists?(name)
|
73
|
+
count = execute_select(GoodData::SQLGenerator.get_table_count(name), :count => true)
|
74
|
+
count > 0
|
75
|
+
end
|
76
|
+
|
77
|
+
def get_columns(table_name)
|
78
|
+
res = execute_select(GoodData::SQLGenerator.get_columns(table_name))
|
79
|
+
end
|
80
|
+
|
81
|
+
# execute sql, return nothing
|
82
|
+
def execute(sql_strings)
|
83
|
+
if ! sql_strings.kind_of?(Array)
|
84
|
+
sql_strings = [sql_strings]
|
85
|
+
end
|
86
|
+
connect do |connection|
|
87
|
+
sql_strings.each do |sql|
|
88
|
+
@logger.info("Executing sql: #{sql}") if @logger
|
89
|
+
connection.run(sql)
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
# executes sql (select), for each row, passes execution to block
|
95
|
+
def execute_select(sql, options={})
|
96
|
+
fetch_handler = options[:fetch_handler]
|
97
|
+
count = options[:count]
|
98
|
+
|
99
|
+
connect do |connection|
|
100
|
+
# do the query
|
101
|
+
f = connection.fetch(sql)
|
102
|
+
|
103
|
+
@logger.info("Executing sql: #{sql}") if @logger
|
104
|
+
# if handler was passed call it
|
105
|
+
if fetch_handler
|
106
|
+
fetch_handler.call(f)
|
107
|
+
end
|
108
|
+
|
109
|
+
if count
|
110
|
+
return f.first[:count]
|
111
|
+
end
|
112
|
+
|
113
|
+
# if block given yield to process line by line
|
114
|
+
if block_given?
|
115
|
+
# go through the rows returned and call the block
|
116
|
+
return f.each do |row|
|
117
|
+
yield(row)
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
# return it all at once
|
122
|
+
f.map{|h| h}
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
def connect
|
127
|
+
Sequel.connect @jdbc_url,
|
128
|
+
:username => @username,
|
129
|
+
:password => @password do |connection|
|
130
|
+
yield(connection)
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
private
|
135
|
+
|
136
|
+
def get_csv_headers(csv_path)
|
137
|
+
header_str = File.open(csv_path, &:gets)
|
138
|
+
if header_str.nil? || header_str.empty?
|
139
|
+
return []
|
140
|
+
end
|
141
|
+
header_str.split(',').map{ |s| s.gsub(/[\s"-]/,'') }
|
142
|
+
end
|
143
|
+
end
|
144
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
module GoodData
|
2
|
+
class SQLGenerator
|
3
|
+
DEFAULT_TYPE = 'varchar(1023)'
|
4
|
+
class << self
|
5
|
+
|
6
|
+
def rename_table(old_name, new_name)
|
7
|
+
"ALTER TABLE #{old_name} RENAME TO #{new_name}"
|
8
|
+
end
|
9
|
+
|
10
|
+
def drop_table(table_name, opts={})
|
11
|
+
"DROP TABLE #{opts[:skip_if_exists] ? 'IF EXISTS' : ''} #{table_name}"
|
12
|
+
end
|
13
|
+
|
14
|
+
def create_table(table_name, columns, opts={})
|
15
|
+
not_exists = opts[:skip_if_exists] ? 'IF NOT EXISTS' : ''
|
16
|
+
columns_string = columns.map { |c|
|
17
|
+
c.is_a?(String) ? "#{c} #{DEFAULT_TYPE}" : "#{c[:column_name]} #{c[:data_type] || DEFAULT_TYPE}"
|
18
|
+
}.join(', ')
|
19
|
+
"CREATE TABLE #{not_exists} #{table_name} (#{columns_string})"
|
20
|
+
end
|
21
|
+
|
22
|
+
def load_data(table, csv, columns)
|
23
|
+
col_list = columns.join(', ')
|
24
|
+
|
25
|
+
# TODO: exceptions, rejections
|
26
|
+
# EXCEPTIONS '#{except_filename(filename)}'
|
27
|
+
# REJECTED DATA '#{reject_filename(filename)}' }
|
28
|
+
|
29
|
+
%Q{COPY #{table} (#{col_list})
|
30
|
+
FROM LOCAL '#{csv}' WITH PARSER GdcCsvParser()
|
31
|
+
ESCAPE AS '"'
|
32
|
+
SKIP 1}
|
33
|
+
end
|
34
|
+
|
35
|
+
def get_table_count(table_name)
|
36
|
+
"SELECT COUNT(*) FROM tables WHERE table_name = '#{table_name}'"
|
37
|
+
end
|
38
|
+
|
39
|
+
def get_columns(table_name)
|
40
|
+
"SELECT column_name, data_type FROM columns WHERE table_name = '#{table_name}'"
|
41
|
+
end
|
42
|
+
|
43
|
+
def select_all(table_name, options={})
|
44
|
+
limit = options[:limit] ? "LIMIT #{options[:limit]}" : ''
|
45
|
+
"SELECT * FROM #{table_name} #{limit}"
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
data/spec/data/bike.csv
ADDED
@@ -0,0 +1,175 @@
|
|
1
|
+
require 'tempfile'
|
2
|
+
require 'gooddata_datawarehouse/datawarehouse'
|
3
|
+
require_relative 'spec_helper'
|
4
|
+
|
5
|
+
describe GoodData::Datawarehouse do
|
6
|
+
before(:each) do
|
7
|
+
@dwh = SpecHelper::create_default_connection
|
8
|
+
@random = rand(10000000).to_s
|
9
|
+
@random_table_name = "temp_#{@random}"
|
10
|
+
@created_tables = nil
|
11
|
+
end
|
12
|
+
|
13
|
+
after(:each) do
|
14
|
+
@created_tables.each{|t| @dwh.drop_table(t)} if @created_tables
|
15
|
+
end
|
16
|
+
|
17
|
+
describe '#create_table' do
|
18
|
+
it 'creates a table with default type' do
|
19
|
+
cols = ['col1', 'col2', 'col3']
|
20
|
+
@dwh.create_table(@random_table_name, cols)
|
21
|
+
@created_tables = [@random_table_name]
|
22
|
+
|
23
|
+
# table exists
|
24
|
+
expect(@dwh.table_exists?(@random_table_name)).to eq true
|
25
|
+
|
26
|
+
# cols are the same
|
27
|
+
expect(Set.new(@dwh.get_columns(@random_table_name))).to eq Set.new(cols.map {|c| {:column_name => c, :data_type => GoodData::SQLGenerator::DEFAULT_TYPE}})
|
28
|
+
end
|
29
|
+
|
30
|
+
it "doesn't create a table when it already exists" do
|
31
|
+
cols = ['col1', 'col2', 'col3']
|
32
|
+
cols2 = ['col1', 'col2']
|
33
|
+
@dwh.create_table(@random_table_name, cols)
|
34
|
+
@created_tables = [@random_table_name]
|
35
|
+
|
36
|
+
expect(@dwh.table_exists?(@random_table_name)).to eq true
|
37
|
+
|
38
|
+
# try to create a table with di
|
39
|
+
@dwh.create_table(@random_table_name, cols2, skip_if_exists: true)
|
40
|
+
|
41
|
+
# table still exists
|
42
|
+
expect(@dwh.table_exists?(@random_table_name)).to eq true
|
43
|
+
# cols are the same
|
44
|
+
expect(Set.new(@dwh.get_columns(@random_table_name))).to eq Set.new(cols.map {|c| {:column_name => c, :data_type => GoodData::SQLGenerator::DEFAULT_TYPE}})
|
45
|
+
end
|
46
|
+
|
47
|
+
it 'creates a table with given types' do
|
48
|
+
cols = [
|
49
|
+
{
|
50
|
+
column_name: 'col1',
|
51
|
+
data_type: 'varchar(88)'
|
52
|
+
}, {
|
53
|
+
column_name: 'col2',
|
54
|
+
data_type: 'int'
|
55
|
+
}, {
|
56
|
+
column_name: 'col3',
|
57
|
+
data_type: 'boolean'
|
58
|
+
}
|
59
|
+
]
|
60
|
+
@dwh.create_table(@random_table_name, cols)
|
61
|
+
@created_tables = [@random_table_name]
|
62
|
+
|
63
|
+
# table exists
|
64
|
+
expect(@dwh.table_exists?(@random_table_name)).to eq true
|
65
|
+
|
66
|
+
# cols are the same
|
67
|
+
expect(Set.new(@dwh.get_columns(@random_table_name))).to eq Set.new(cols)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
describe '#drop_table' do
|
72
|
+
it 'drops a table' do
|
73
|
+
cols = ['col1', 'col2', 'col3']
|
74
|
+
|
75
|
+
@dwh.create_table(@random_table_name, cols)
|
76
|
+
expect(@dwh.table_exists?(@random_table_name)).to eq true
|
77
|
+
|
78
|
+
# it shouldn't exist after being dropped
|
79
|
+
@dwh.drop_table(@random_table_name)
|
80
|
+
expect(@dwh.table_exists?(@random_table_name)).to eq false
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
describe '#rename_table' do
|
85
|
+
it 'renames a table' do
|
86
|
+
cols = ['col1', 'col2', 'col3']
|
87
|
+
|
88
|
+
@dwh.create_table(@random_table_name, cols)
|
89
|
+
expect(@dwh.table_exists?(@random_table_name)).to eq true
|
90
|
+
|
91
|
+
# the renamed table should exist, not the old name
|
92
|
+
changed_name = "#{@random_table_name}_something"
|
93
|
+
@dwh.rename_table(@random_table_name, changed_name)
|
94
|
+
expect(@dwh.table_exists?(@random_table_name)).to eq false
|
95
|
+
expect(@dwh.table_exists?(changed_name)).to eq true
|
96
|
+
|
97
|
+
@created_tables = [changed_name]
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
describe '#csv_to_new_table' do
|
102
|
+
it 'creates a new table from csv' do
|
103
|
+
path = 'spec/data/bike.csv'
|
104
|
+
@dwh.csv_to_new_table(@random_table_name, path)
|
105
|
+
|
106
|
+
# table exists
|
107
|
+
expect(@dwh.table_exists?(@random_table_name)).to eq true
|
108
|
+
|
109
|
+
# cols are the same as in the csv
|
110
|
+
expected_cols = File.open(path, &:gets).strip.split(',')
|
111
|
+
expect(Set.new(@dwh.get_columns(@random_table_name))).to eq Set.new(expected_cols.map {|c| {:column_name => c, :data_type => GoodData::SQLGenerator::DEFAULT_TYPE}})
|
112
|
+
@created_tables = [@random_table_name]
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
describe '#export_table' do
|
117
|
+
it 'exports a created table' do
|
118
|
+
path = 'spec/data/bike.csv'
|
119
|
+
@dwh.csv_to_new_table(@random_table_name, path)
|
120
|
+
|
121
|
+
# table exists
|
122
|
+
expect(@dwh.table_exists?(@random_table_name)).to eq true
|
123
|
+
|
124
|
+
# export it
|
125
|
+
f = Tempfile.new('bike.csv')
|
126
|
+
@dwh.export_table(@random_table_name, f)
|
127
|
+
|
128
|
+
# should be the same except for order of the lines
|
129
|
+
imported = Set.new(CSV.read(path))
|
130
|
+
exported = Set.new(CSV.read(f))
|
131
|
+
|
132
|
+
expect(exported).to eq imported
|
133
|
+
@created_tables = [@random_table_name]
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
describe '#load_data_from_csv' do
|
138
|
+
it 'loads data from csv to existing table' do
|
139
|
+
path = 'spec/data/bike.csv'
|
140
|
+
|
141
|
+
# create the table
|
142
|
+
@dwh.create_table_from_csv_header(@random_table_name, path)
|
143
|
+
expect(@dwh.table_exists?(@random_table_name)).to eq true
|
144
|
+
|
145
|
+
expected_cols = File.open(path, &:gets).strip.split(',')
|
146
|
+
expect(Set.new(@dwh.get_columns(@random_table_name))).to eq Set.new(expected_cols.map {|c| {:column_name => c, :data_type => GoodData::SQLGenerator::DEFAULT_TYPE}})
|
147
|
+
|
148
|
+
# load the data there
|
149
|
+
@dwh.load_data_from_csv(@random_table_name, path)
|
150
|
+
|
151
|
+
# export it
|
152
|
+
f = Tempfile.new('bike.csv')
|
153
|
+
@dwh.export_table(@random_table_name, f)
|
154
|
+
|
155
|
+
# should be the same except for order of the lines
|
156
|
+
imported = Set.new(CSV.read(path))
|
157
|
+
exported = Set.new(CSV.read(f))
|
158
|
+
|
159
|
+
expect(exported).to eq imported
|
160
|
+
@created_tables = [@random_table_name]
|
161
|
+
end
|
162
|
+
|
163
|
+
it 'fails for a wrong csv' do
|
164
|
+
path_wrong = 'spec/data/bike-wrong.csv'
|
165
|
+
|
166
|
+
# create the table
|
167
|
+
@dwh.create_table_from_csv_header(@random_table_name, path_wrong)
|
168
|
+
expect(@dwh.table_exists?(@random_table_name)).to eq true
|
169
|
+
|
170
|
+
# load the data there
|
171
|
+
@dwh.load_data_from_csv(@random_table_name, path_wrong)
|
172
|
+
|
173
|
+
end
|
174
|
+
end
|
175
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
require 'gooddata_datawarehouse'
|
2
|
+
|
3
|
+
RSpec.configure do |c|
|
4
|
+
c.filter_run :focus => true
|
5
|
+
c.run_all_when_everything_filtered = true
|
6
|
+
end
|
7
|
+
|
8
|
+
class SpecHelper
|
9
|
+
def self.create_default_connection
|
10
|
+
GoodData::Datawarehouse.new(ENV['USERNAME'], ENV['PASSWORD'], ENV['INSTANCE_ID'])
|
11
|
+
end
|
12
|
+
end
|
metadata
ADDED
@@ -0,0 +1,145 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: gooddata_datawarehouse
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Petr Cvengros
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2014-12-18 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
15
|
+
requirements:
|
16
|
+
- - ~>
|
17
|
+
- !ruby/object:Gem::Version
|
18
|
+
version: '1.7'
|
19
|
+
name: bundler
|
20
|
+
prerelease: false
|
21
|
+
type: :development
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ~>
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.7'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
requirement: !ruby/object:Gem::Requirement
|
29
|
+
requirements:
|
30
|
+
- - ~>
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '10.0'
|
33
|
+
name: rake
|
34
|
+
prerelease: false
|
35
|
+
type: :development
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ~>
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '10.0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
requirement: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - ~>
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: '2.14'
|
47
|
+
name: rspec
|
48
|
+
prerelease: false
|
49
|
+
type: :development
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ~>
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '2.14'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
requirement: !ruby/object:Gem::Requirement
|
57
|
+
requirements:
|
58
|
+
- - ~>
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: '0.9'
|
61
|
+
name: pry
|
62
|
+
prerelease: false
|
63
|
+
type: :development
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ~>
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0.9'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
requirement: !ruby/object:Gem::Requirement
|
71
|
+
requirements:
|
72
|
+
- - ~>
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: '4.17'
|
75
|
+
name: sequel
|
76
|
+
prerelease: false
|
77
|
+
type: :runtime
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ~>
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '4.17'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
requirement: !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - ~>
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: '0.1'
|
89
|
+
name: gooddata-dss-jdbc
|
90
|
+
prerelease: false
|
91
|
+
type: :runtime
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ~>
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0.1'
|
97
|
+
description: ''
|
98
|
+
email:
|
99
|
+
- petr.cvengros@gooddata.com
|
100
|
+
executables: []
|
101
|
+
extensions: []
|
102
|
+
extra_rdoc_files: []
|
103
|
+
files:
|
104
|
+
- .gitignore
|
105
|
+
- Gemfile
|
106
|
+
- LICENSE.txt
|
107
|
+
- README.md
|
108
|
+
- Rakefile
|
109
|
+
- env_setup-example.sh
|
110
|
+
- gooddata_datawarehouse.gemspec
|
111
|
+
- lib/gooddata_datawarehouse.rb
|
112
|
+
- lib/gooddata_datawarehouse/datawarehouse.rb
|
113
|
+
- lib/gooddata_datawarehouse/sql_generator.rb
|
114
|
+
- lib/gooddata_datawarehouse/version.rb
|
115
|
+
- spec/data/bike.csv
|
116
|
+
- spec/datawarehouse_spec.rb
|
117
|
+
- spec/spec_helper.rb
|
118
|
+
homepage: ''
|
119
|
+
licenses:
|
120
|
+
- MIT
|
121
|
+
metadata: {}
|
122
|
+
post_install_message:
|
123
|
+
rdoc_options: []
|
124
|
+
require_paths:
|
125
|
+
- lib
|
126
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
127
|
+
requirements:
|
128
|
+
- - '>='
|
129
|
+
- !ruby/object:Gem::Version
|
130
|
+
version: '0'
|
131
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
132
|
+
requirements:
|
133
|
+
- - '>='
|
134
|
+
- !ruby/object:Gem::Version
|
135
|
+
version: '0'
|
136
|
+
requirements: []
|
137
|
+
rubyforge_project:
|
138
|
+
rubygems_version: 2.1.9
|
139
|
+
signing_key:
|
140
|
+
specification_version: 4
|
141
|
+
summary: Convenient work with GoodData's Datawarehouse (ADS)
|
142
|
+
test_files:
|
143
|
+
- spec/data/bike.csv
|
144
|
+
- spec/datawarehouse_spec.rb
|
145
|
+
- spec/spec_helper.rb
|