gooddata_datawarehouse 0.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +36 -0
- data/Gemfile +4 -0
- data/LICENSE.txt +22 -0
- data/README.md +61 -0
- data/Rakefile +2 -0
- data/env_setup-example.sh +5 -0
- data/gooddata_datawarehouse.gemspec +32 -0
- data/lib/gooddata_datawarehouse/datawarehouse.rb +144 -0
- data/lib/gooddata_datawarehouse/sql_generator.rb +49 -0
- data/lib/gooddata_datawarehouse/version.rb +5 -0
- data/lib/gooddata_datawarehouse.rb +2 -0
- data/spec/data/bike.csv +5 -0
- data/spec/datawarehouse_spec.rb +175 -0
- data/spec/spec_helper.rb +12 -0
- metadata +145 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 6a3ba4f26ae8e33a6118a1f0a892315968f92857
|
4
|
+
data.tar.gz: 2be9b35b829f8910116a65252a3651ab6dfd5be6
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: c2b74588cb194e9c0928dcd3c6c3250bf64907d1fe4fc93d137506a2f2f8e86c02350cbdbbc7341c5854a30d75ac7ad36b2269e8f3c5f393959f6b4ee1b26b59
|
7
|
+
data.tar.gz: e7e8b1cb11d8926b47d516e9a636b79096d055bb7ca35331bb6edcf5d67926fde3a57ebb32c719741f83cf0caa2e6da66f2f0688535be91e13158e55af446113
|
data/.gitignore
ADDED
@@ -0,0 +1,36 @@
|
|
1
|
+
*.gem
|
2
|
+
*.rbc
|
3
|
+
/.config
|
4
|
+
/coverage/
|
5
|
+
/InstalledFiles
|
6
|
+
/pkg/
|
7
|
+
/spec/reports/
|
8
|
+
/test/tmp/
|
9
|
+
/test/version_tmp/
|
10
|
+
/tmp/
|
11
|
+
|
12
|
+
## Specific to RubyMotion:
|
13
|
+
.dat*
|
14
|
+
.repl_history
|
15
|
+
build/
|
16
|
+
|
17
|
+
## Documentation cache and generated files:
|
18
|
+
/.yardoc/
|
19
|
+
/_yardoc/
|
20
|
+
/doc/
|
21
|
+
/rdoc/
|
22
|
+
|
23
|
+
## Environment normalisation:
|
24
|
+
/.bundle/
|
25
|
+
/lib/bundler/man/
|
26
|
+
|
27
|
+
# for a library or gem, you might want to ignore these files since the code is
|
28
|
+
# intended to run in multiple environments; otherwise, check them in:
|
29
|
+
Gemfile.lock
|
30
|
+
.ruby-version
|
31
|
+
.ruby-gemset
|
32
|
+
|
33
|
+
# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
|
34
|
+
.rvmrc
|
35
|
+
|
36
|
+
env_setup.sh
|
data/Gemfile
ADDED
data/LICENSE.txt
ADDED
@@ -0,0 +1,22 @@
|
|
1
|
+
BSD License
|
2
|
+
Copyright (c) 2013, GoodData Corporation
|
3
|
+
Redistribution and use in source and binary forms, with or without
|
4
|
+
modification, are permitted provided that the following conditions are met:
|
5
|
+
* Redistributions of source code must retain the above copyright notice,
|
6
|
+
this list of conditions and the following disclaimer.
|
7
|
+
* Redistributions in binary form must reproduce the above copyright notice,
|
8
|
+
this list of conditions and the following disclaimer in the documentation
|
9
|
+
and/or other materials provided with the distribution.
|
10
|
+
* Neither the name of GoodData Corporation nor the names of its contributors
|
11
|
+
may be used to endorse or promote products derived from this software
|
12
|
+
without specific prior written permission.
|
13
|
+
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
14
|
+
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
|
15
|
+
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
|
16
|
+
ARE DISCLAIMED. IN NO EVENT SHALL GOODDATA CORPORATION BE LIABLE FOR ANY
|
17
|
+
DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
18
|
+
(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
19
|
+
LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
20
|
+
ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
21
|
+
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
|
22
|
+
THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
data/README.md
ADDED
@@ -0,0 +1,61 @@
|
|
1
|
+
# GooddataDatawarehouse
|
2
|
+
|
3
|
+
A little library to help you work with GoodData's Datawarehouse
|
4
|
+
|
5
|
+
## Installation
|
6
|
+
|
7
|
+
Add this line to your application's Gemfile:
|
8
|
+
|
9
|
+
```ruby
|
10
|
+
gem 'gooddata_datawarehouse'
|
11
|
+
```
|
12
|
+
|
13
|
+
And then execute:
|
14
|
+
|
15
|
+
$ bundle
|
16
|
+
|
17
|
+
Or install it yourself as:
|
18
|
+
|
19
|
+
$ gem install gooddata_datawarehouse
|
20
|
+
|
21
|
+
## Usage
|
22
|
+
|
23
|
+
```ruby
|
24
|
+
require 'gooddata_datawarehouse'
|
25
|
+
|
26
|
+
# connect
|
27
|
+
dwh = GoodData::Datawarehouse.new('you@gooddata.com', 'yourpass', 'your ADS instance id'
|
28
|
+
|
29
|
+
# import a csv
|
30
|
+
dwh.csv_to_new_table('my_table', 'path/to/my.csv')
|
31
|
+
|
32
|
+
dwh.table_exists?('my_table') # true
|
33
|
+
dwh.get_columns('my_table') # [{column_name: 'col1', data_type: 'varchar(88)'}, {column_name: 'col2', data_type: 'int'}]
|
34
|
+
|
35
|
+
# run an aribrary sql
|
36
|
+
dwh.execute('ALTER TABLE my_table ADD COLUMN col3 INTEGER')
|
37
|
+
|
38
|
+
# run a select and process results
|
39
|
+
dwh.execute_select('SELECT * FROM my_table ORDER BY col1') do |row|
|
40
|
+
puts row[:col1]
|
41
|
+
end
|
42
|
+
|
43
|
+
|
44
|
+
# rename a table
|
45
|
+
dwh.rename_table('my_table', 'my_new_table')
|
46
|
+
|
47
|
+
# export to csv
|
48
|
+
dwh.export_table('my_new_table', 'path/to/my_new.csv')
|
49
|
+
|
50
|
+
# drop table
|
51
|
+
dwh.drop_table('my_new_table')
|
52
|
+
```
|
53
|
+
|
54
|
+
|
55
|
+
## Contributing
|
56
|
+
|
57
|
+
1. Fork it ( https://github.com/[my-github-username]/gooddata_datawarehouse/fork )
|
58
|
+
2. Create your feature branch (`git checkout -b my-new-feature`)
|
59
|
+
3. Commit your changes (`git commit -am 'Add some feature'`)
|
60
|
+
4. Push to the branch (`git push origin my-new-feature`)
|
61
|
+
5. Create a new Pull Request
|
data/Rakefile
ADDED
@@ -0,0 +1,32 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'gooddata_datawarehouse/version'
|
5
|
+
|
6
|
+
if RUBY_PLATFORM != 'java'
|
7
|
+
fail "Only java platform supported! Use jRuby, e.g. rvm use jruby"
|
8
|
+
end
|
9
|
+
|
10
|
+
Gem::Specification.new do |spec|
|
11
|
+
spec.name = "gooddata_datawarehouse"
|
12
|
+
spec.version = GoodData::Datawarehouse::VERSION
|
13
|
+
spec.authors = ["Petr Cvengros"]
|
14
|
+
spec.email = ["petr.cvengros@gooddata.com"]
|
15
|
+
spec.summary = %q{Convenient work with GoodData's Datawarehouse (ADS) }
|
16
|
+
spec.description = ""
|
17
|
+
spec.homepage = ""
|
18
|
+
spec.license = "MIT"
|
19
|
+
|
20
|
+
spec.files = `git ls-files -z`.split("\x0")
|
21
|
+
spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
22
|
+
spec.test_files = spec.files.grep(%r{^(test|spec|features)/})
|
23
|
+
spec.require_paths = ["lib"]
|
24
|
+
|
25
|
+
spec.add_development_dependency "bundler", "~> 1.7"
|
26
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
27
|
+
spec.add_development_dependency 'rspec', '~>2.14'
|
28
|
+
spec.add_development_dependency 'pry', '~> 0.9'
|
29
|
+
|
30
|
+
spec.add_dependency "sequel", "~> 4.17"
|
31
|
+
spec.add_dependency "gooddata-dss-jdbc", "~> 0.1"
|
32
|
+
end
|
@@ -0,0 +1,144 @@
|
|
1
|
+
require 'jdbc/dss'
|
2
|
+
require 'sequel'
|
3
|
+
require 'logger'
|
4
|
+
require 'csv'
|
5
|
+
|
6
|
+
require_relative 'sql_generator'
|
7
|
+
|
8
|
+
module GoodData
|
9
|
+
class Datawarehouse
|
10
|
+
def initialize(username, password, instance_id, options={})
|
11
|
+
@logger = Logger.new(STDOUT)
|
12
|
+
@username = username
|
13
|
+
@password = password
|
14
|
+
@jdbc_url = "jdbc:dss://secure.gooddata.com/gdc/dss/instances/#{instance_id}"
|
15
|
+
Jdbc::DSS.load_driver
|
16
|
+
Java.com.gooddata.dss.jdbc.driver.DssDriver
|
17
|
+
end
|
18
|
+
|
19
|
+
def export_table(table_name, csv_path)
|
20
|
+
CSV.open(csv_path, 'wb', :force_quotes => true) do |csv|
|
21
|
+
# get the names of cols
|
22
|
+
cols = get_columns(table_name).map {|c| c[:column_name]}
|
23
|
+
col_names =
|
24
|
+
|
25
|
+
# write header
|
26
|
+
csv << cols
|
27
|
+
|
28
|
+
# get the keys for columns, stupid sequel
|
29
|
+
col_keys = nil
|
30
|
+
rows = execute_select(GoodData::SQLGenerator.select_all(table_name, limit: 1))
|
31
|
+
|
32
|
+
col_keys = rows[0].keys
|
33
|
+
|
34
|
+
execute_select(GoodData::SQLGenerator.select_all(table_name)) do |row|
|
35
|
+
# go through the table write to csv
|
36
|
+
csv << row.values_at(*col_keys)
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
def rename_table(old_name, new_name)
|
42
|
+
execute(GoodData::SQLGenerator.rename_table(old_name, new_name))
|
43
|
+
end
|
44
|
+
|
45
|
+
def drop_table(table_name, opts={})
|
46
|
+
execute(GoodData::SQLGenerator.drop_table(table_name,opts))
|
47
|
+
end
|
48
|
+
|
49
|
+
def csv_to_new_table(table_name, csv_path, opts={})
|
50
|
+
cols = create_table_from_csv_header(table_name, csv_path, opts)
|
51
|
+
load_data_from_csv(table_name, csv_path, columns: cols)
|
52
|
+
end
|
53
|
+
|
54
|
+
def load_data_from_csv(table_name, csv_path, opts={})
|
55
|
+
columns = opts[:columns] || get_csv_headers(csv_path)
|
56
|
+
execute(GoodData::SQLGenerator.load_data(table_name, csv_path, columns))
|
57
|
+
end
|
58
|
+
|
59
|
+
# returns a list of columns created
|
60
|
+
# does nothing if file empty, returns []
|
61
|
+
def create_table_from_csv_header(table_name, csv_path, opts={})
|
62
|
+
# take the header as a list of columns
|
63
|
+
columns = get_csv_headers(csv_path)
|
64
|
+
create_table(table_name, columns, opts) unless columns.empty?
|
65
|
+
columns
|
66
|
+
end
|
67
|
+
|
68
|
+
def create_table(name, columns, options={})
|
69
|
+
execute(GoodData::SQLGenerator.create_table(name, columns, options))
|
70
|
+
end
|
71
|
+
|
72
|
+
def table_exists?(name)
|
73
|
+
count = execute_select(GoodData::SQLGenerator.get_table_count(name), :count => true)
|
74
|
+
count > 0
|
75
|
+
end
|
76
|
+
|
77
|
+
def get_columns(table_name)
|
78
|
+
res = execute_select(GoodData::SQLGenerator.get_columns(table_name))
|
79
|
+
end
|
80
|
+
|
81
|
+
# execute sql, return nothing
|
82
|
+
def execute(sql_strings)
|
83
|
+
if ! sql_strings.kind_of?(Array)
|
84
|
+
sql_strings = [sql_strings]
|
85
|
+
end
|
86
|
+
connect do |connection|
|
87
|
+
sql_strings.each do |sql|
|
88
|
+
@logger.info("Executing sql: #{sql}") if @logger
|
89
|
+
connection.run(sql)
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
# executes sql (select), for each row, passes execution to block
|
95
|
+
def execute_select(sql, options={})
|
96
|
+
fetch_handler = options[:fetch_handler]
|
97
|
+
count = options[:count]
|
98
|
+
|
99
|
+
connect do |connection|
|
100
|
+
# do the query
|
101
|
+
f = connection.fetch(sql)
|
102
|
+
|
103
|
+
@logger.info("Executing sql: #{sql}") if @logger
|
104
|
+
# if handler was passed call it
|
105
|
+
if fetch_handler
|
106
|
+
fetch_handler.call(f)
|
107
|
+
end
|
108
|
+
|
109
|
+
if count
|
110
|
+
return f.first[:count]
|
111
|
+
end
|
112
|
+
|
113
|
+
# if block given yield to process line by line
|
114
|
+
if block_given?
|
115
|
+
# go through the rows returned and call the block
|
116
|
+
return f.each do |row|
|
117
|
+
yield(row)
|
118
|
+
end
|
119
|
+
end
|
120
|
+
|
121
|
+
# return it all at once
|
122
|
+
f.map{|h| h}
|
123
|
+
end
|
124
|
+
end
|
125
|
+
|
126
|
+
def connect
|
127
|
+
Sequel.connect @jdbc_url,
|
128
|
+
:username => @username,
|
129
|
+
:password => @password do |connection|
|
130
|
+
yield(connection)
|
131
|
+
end
|
132
|
+
end
|
133
|
+
|
134
|
+
private
|
135
|
+
|
136
|
+
def get_csv_headers(csv_path)
|
137
|
+
header_str = File.open(csv_path, &:gets)
|
138
|
+
if header_str.nil? || header_str.empty?
|
139
|
+
return []
|
140
|
+
end
|
141
|
+
header_str.split(',').map{ |s| s.gsub(/[\s"-]/,'') }
|
142
|
+
end
|
143
|
+
end
|
144
|
+
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
module GoodData
|
2
|
+
class SQLGenerator
|
3
|
+
DEFAULT_TYPE = 'varchar(1023)'
|
4
|
+
class << self
|
5
|
+
|
6
|
+
def rename_table(old_name, new_name)
|
7
|
+
"ALTER TABLE #{old_name} RENAME TO #{new_name}"
|
8
|
+
end
|
9
|
+
|
10
|
+
def drop_table(table_name, opts={})
|
11
|
+
"DROP TABLE #{opts[:skip_if_exists] ? 'IF EXISTS' : ''} #{table_name}"
|
12
|
+
end
|
13
|
+
|
14
|
+
def create_table(table_name, columns, opts={})
|
15
|
+
not_exists = opts[:skip_if_exists] ? 'IF NOT EXISTS' : ''
|
16
|
+
columns_string = columns.map { |c|
|
17
|
+
c.is_a?(String) ? "#{c} #{DEFAULT_TYPE}" : "#{c[:column_name]} #{c[:data_type] || DEFAULT_TYPE}"
|
18
|
+
}.join(', ')
|
19
|
+
"CREATE TABLE #{not_exists} #{table_name} (#{columns_string})"
|
20
|
+
end
|
21
|
+
|
22
|
+
def load_data(table, csv, columns)
|
23
|
+
col_list = columns.join(', ')
|
24
|
+
|
25
|
+
# TODO: exceptions, rejections
|
26
|
+
# EXCEPTIONS '#{except_filename(filename)}'
|
27
|
+
# REJECTED DATA '#{reject_filename(filename)}' }
|
28
|
+
|
29
|
+
%Q{COPY #{table} (#{col_list})
|
30
|
+
FROM LOCAL '#{csv}' WITH PARSER GdcCsvParser()
|
31
|
+
ESCAPE AS '"'
|
32
|
+
SKIP 1}
|
33
|
+
end
|
34
|
+
|
35
|
+
def get_table_count(table_name)
|
36
|
+
"SELECT COUNT(*) FROM tables WHERE table_name = '#{table_name}'"
|
37
|
+
end
|
38
|
+
|
39
|
+
def get_columns(table_name)
|
40
|
+
"SELECT column_name, data_type FROM columns WHERE table_name = '#{table_name}'"
|
41
|
+
end
|
42
|
+
|
43
|
+
def select_all(table_name, options={})
|
44
|
+
limit = options[:limit] ? "LIMIT #{options[:limit]}" : ''
|
45
|
+
"SELECT * FROM #{table_name} #{limit}"
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
data/spec/data/bike.csv
ADDED
@@ -0,0 +1,175 @@
|
|
1
|
+
require 'tempfile'
|
2
|
+
require 'gooddata_datawarehouse/datawarehouse'
|
3
|
+
require_relative 'spec_helper'
|
4
|
+
|
5
|
+
describe GoodData::Datawarehouse do
|
6
|
+
before(:each) do
|
7
|
+
@dwh = SpecHelper::create_default_connection
|
8
|
+
@random = rand(10000000).to_s
|
9
|
+
@random_table_name = "temp_#{@random}"
|
10
|
+
@created_tables = nil
|
11
|
+
end
|
12
|
+
|
13
|
+
after(:each) do
|
14
|
+
@created_tables.each{|t| @dwh.drop_table(t)} if @created_tables
|
15
|
+
end
|
16
|
+
|
17
|
+
describe '#create_table' do
|
18
|
+
it 'creates a table with default type' do
|
19
|
+
cols = ['col1', 'col2', 'col3']
|
20
|
+
@dwh.create_table(@random_table_name, cols)
|
21
|
+
@created_tables = [@random_table_name]
|
22
|
+
|
23
|
+
# table exists
|
24
|
+
expect(@dwh.table_exists?(@random_table_name)).to eq true
|
25
|
+
|
26
|
+
# cols are the same
|
27
|
+
expect(Set.new(@dwh.get_columns(@random_table_name))).to eq Set.new(cols.map {|c| {:column_name => c, :data_type => GoodData::SQLGenerator::DEFAULT_TYPE}})
|
28
|
+
end
|
29
|
+
|
30
|
+
it "doesn't create a table when it already exists" do
|
31
|
+
cols = ['col1', 'col2', 'col3']
|
32
|
+
cols2 = ['col1', 'col2']
|
33
|
+
@dwh.create_table(@random_table_name, cols)
|
34
|
+
@created_tables = [@random_table_name]
|
35
|
+
|
36
|
+
expect(@dwh.table_exists?(@random_table_name)).to eq true
|
37
|
+
|
38
|
+
# try to create a table with di
|
39
|
+
@dwh.create_table(@random_table_name, cols2, skip_if_exists: true)
|
40
|
+
|
41
|
+
# table still exists
|
42
|
+
expect(@dwh.table_exists?(@random_table_name)).to eq true
|
43
|
+
# cols are the same
|
44
|
+
expect(Set.new(@dwh.get_columns(@random_table_name))).to eq Set.new(cols.map {|c| {:column_name => c, :data_type => GoodData::SQLGenerator::DEFAULT_TYPE}})
|
45
|
+
end
|
46
|
+
|
47
|
+
it 'creates a table with given types' do
|
48
|
+
cols = [
|
49
|
+
{
|
50
|
+
column_name: 'col1',
|
51
|
+
data_type: 'varchar(88)'
|
52
|
+
}, {
|
53
|
+
column_name: 'col2',
|
54
|
+
data_type: 'int'
|
55
|
+
}, {
|
56
|
+
column_name: 'col3',
|
57
|
+
data_type: 'boolean'
|
58
|
+
}
|
59
|
+
]
|
60
|
+
@dwh.create_table(@random_table_name, cols)
|
61
|
+
@created_tables = [@random_table_name]
|
62
|
+
|
63
|
+
# table exists
|
64
|
+
expect(@dwh.table_exists?(@random_table_name)).to eq true
|
65
|
+
|
66
|
+
# cols are the same
|
67
|
+
expect(Set.new(@dwh.get_columns(@random_table_name))).to eq Set.new(cols)
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
describe '#drop_table' do
|
72
|
+
it 'drops a table' do
|
73
|
+
cols = ['col1', 'col2', 'col3']
|
74
|
+
|
75
|
+
@dwh.create_table(@random_table_name, cols)
|
76
|
+
expect(@dwh.table_exists?(@random_table_name)).to eq true
|
77
|
+
|
78
|
+
# it shouldn't exist after being dropped
|
79
|
+
@dwh.drop_table(@random_table_name)
|
80
|
+
expect(@dwh.table_exists?(@random_table_name)).to eq false
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
describe '#rename_table' do
|
85
|
+
it 'renames a table' do
|
86
|
+
cols = ['col1', 'col2', 'col3']
|
87
|
+
|
88
|
+
@dwh.create_table(@random_table_name, cols)
|
89
|
+
expect(@dwh.table_exists?(@random_table_name)).to eq true
|
90
|
+
|
91
|
+
# the renamed table should exist, not the old name
|
92
|
+
changed_name = "#{@random_table_name}_something"
|
93
|
+
@dwh.rename_table(@random_table_name, changed_name)
|
94
|
+
expect(@dwh.table_exists?(@random_table_name)).to eq false
|
95
|
+
expect(@dwh.table_exists?(changed_name)).to eq true
|
96
|
+
|
97
|
+
@created_tables = [changed_name]
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
describe '#csv_to_new_table' do
|
102
|
+
it 'creates a new table from csv' do
|
103
|
+
path = 'spec/data/bike.csv'
|
104
|
+
@dwh.csv_to_new_table(@random_table_name, path)
|
105
|
+
|
106
|
+
# table exists
|
107
|
+
expect(@dwh.table_exists?(@random_table_name)).to eq true
|
108
|
+
|
109
|
+
# cols are the same as in the csv
|
110
|
+
expected_cols = File.open(path, &:gets).strip.split(',')
|
111
|
+
expect(Set.new(@dwh.get_columns(@random_table_name))).to eq Set.new(expected_cols.map {|c| {:column_name => c, :data_type => GoodData::SQLGenerator::DEFAULT_TYPE}})
|
112
|
+
@created_tables = [@random_table_name]
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
describe '#export_table' do
|
117
|
+
it 'exports a created table' do
|
118
|
+
path = 'spec/data/bike.csv'
|
119
|
+
@dwh.csv_to_new_table(@random_table_name, path)
|
120
|
+
|
121
|
+
# table exists
|
122
|
+
expect(@dwh.table_exists?(@random_table_name)).to eq true
|
123
|
+
|
124
|
+
# export it
|
125
|
+
f = Tempfile.new('bike.csv')
|
126
|
+
@dwh.export_table(@random_table_name, f)
|
127
|
+
|
128
|
+
# should be the same except for order of the lines
|
129
|
+
imported = Set.new(CSV.read(path))
|
130
|
+
exported = Set.new(CSV.read(f))
|
131
|
+
|
132
|
+
expect(exported).to eq imported
|
133
|
+
@created_tables = [@random_table_name]
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
describe '#load_data_from_csv' do
|
138
|
+
it 'loads data from csv to existing table' do
|
139
|
+
path = 'spec/data/bike.csv'
|
140
|
+
|
141
|
+
# create the table
|
142
|
+
@dwh.create_table_from_csv_header(@random_table_name, path)
|
143
|
+
expect(@dwh.table_exists?(@random_table_name)).to eq true
|
144
|
+
|
145
|
+
expected_cols = File.open(path, &:gets).strip.split(',')
|
146
|
+
expect(Set.new(@dwh.get_columns(@random_table_name))).to eq Set.new(expected_cols.map {|c| {:column_name => c, :data_type => GoodData::SQLGenerator::DEFAULT_TYPE}})
|
147
|
+
|
148
|
+
# load the data there
|
149
|
+
@dwh.load_data_from_csv(@random_table_name, path)
|
150
|
+
|
151
|
+
# export it
|
152
|
+
f = Tempfile.new('bike.csv')
|
153
|
+
@dwh.export_table(@random_table_name, f)
|
154
|
+
|
155
|
+
# should be the same except for order of the lines
|
156
|
+
imported = Set.new(CSV.read(path))
|
157
|
+
exported = Set.new(CSV.read(f))
|
158
|
+
|
159
|
+
expect(exported).to eq imported
|
160
|
+
@created_tables = [@random_table_name]
|
161
|
+
end
|
162
|
+
|
163
|
+
it 'fails for a wrong csv' do
|
164
|
+
path_wrong = 'spec/data/bike-wrong.csv'
|
165
|
+
|
166
|
+
# create the table
|
167
|
+
@dwh.create_table_from_csv_header(@random_table_name, path_wrong)
|
168
|
+
expect(@dwh.table_exists?(@random_table_name)).to eq true
|
169
|
+
|
170
|
+
# load the data there
|
171
|
+
@dwh.load_data_from_csv(@random_table_name, path_wrong)
|
172
|
+
|
173
|
+
end
|
174
|
+
end
|
175
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,12 @@
|
|
1
|
+
require 'gooddata_datawarehouse'
|
2
|
+
|
3
|
+
RSpec.configure do |c|
|
4
|
+
c.filter_run :focus => true
|
5
|
+
c.run_all_when_everything_filtered = true
|
6
|
+
end
|
7
|
+
|
8
|
+
class SpecHelper
|
9
|
+
def self.create_default_connection
|
10
|
+
GoodData::Datawarehouse.new(ENV['USERNAME'], ENV['PASSWORD'], ENV['INSTANCE_ID'])
|
11
|
+
end
|
12
|
+
end
|
metadata
ADDED
@@ -0,0 +1,145 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: gooddata_datawarehouse
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.0.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Petr Cvengros
|
8
|
+
autorequire:
|
9
|
+
bindir: bin
|
10
|
+
cert_chain: []
|
11
|
+
date: 2014-12-18 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
15
|
+
requirements:
|
16
|
+
- - ~>
|
17
|
+
- !ruby/object:Gem::Version
|
18
|
+
version: '1.7'
|
19
|
+
name: bundler
|
20
|
+
prerelease: false
|
21
|
+
type: :development
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - ~>
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.7'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
requirement: !ruby/object:Gem::Requirement
|
29
|
+
requirements:
|
30
|
+
- - ~>
|
31
|
+
- !ruby/object:Gem::Version
|
32
|
+
version: '10.0'
|
33
|
+
name: rake
|
34
|
+
prerelease: false
|
35
|
+
type: :development
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ~>
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '10.0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
requirement: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - ~>
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: '2.14'
|
47
|
+
name: rspec
|
48
|
+
prerelease: false
|
49
|
+
type: :development
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - ~>
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '2.14'
|
55
|
+
- !ruby/object:Gem::Dependency
|
56
|
+
requirement: !ruby/object:Gem::Requirement
|
57
|
+
requirements:
|
58
|
+
- - ~>
|
59
|
+
- !ruby/object:Gem::Version
|
60
|
+
version: '0.9'
|
61
|
+
name: pry
|
62
|
+
prerelease: false
|
63
|
+
type: :development
|
64
|
+
version_requirements: !ruby/object:Gem::Requirement
|
65
|
+
requirements:
|
66
|
+
- - ~>
|
67
|
+
- !ruby/object:Gem::Version
|
68
|
+
version: '0.9'
|
69
|
+
- !ruby/object:Gem::Dependency
|
70
|
+
requirement: !ruby/object:Gem::Requirement
|
71
|
+
requirements:
|
72
|
+
- - ~>
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: '4.17'
|
75
|
+
name: sequel
|
76
|
+
prerelease: false
|
77
|
+
type: :runtime
|
78
|
+
version_requirements: !ruby/object:Gem::Requirement
|
79
|
+
requirements:
|
80
|
+
- - ~>
|
81
|
+
- !ruby/object:Gem::Version
|
82
|
+
version: '4.17'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
requirement: !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - ~>
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: '0.1'
|
89
|
+
name: gooddata-dss-jdbc
|
90
|
+
prerelease: false
|
91
|
+
type: :runtime
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ~>
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0.1'
|
97
|
+
description: ''
|
98
|
+
email:
|
99
|
+
- petr.cvengros@gooddata.com
|
100
|
+
executables: []
|
101
|
+
extensions: []
|
102
|
+
extra_rdoc_files: []
|
103
|
+
files:
|
104
|
+
- .gitignore
|
105
|
+
- Gemfile
|
106
|
+
- LICENSE.txt
|
107
|
+
- README.md
|
108
|
+
- Rakefile
|
109
|
+
- env_setup-example.sh
|
110
|
+
- gooddata_datawarehouse.gemspec
|
111
|
+
- lib/gooddata_datawarehouse.rb
|
112
|
+
- lib/gooddata_datawarehouse/datawarehouse.rb
|
113
|
+
- lib/gooddata_datawarehouse/sql_generator.rb
|
114
|
+
- lib/gooddata_datawarehouse/version.rb
|
115
|
+
- spec/data/bike.csv
|
116
|
+
- spec/datawarehouse_spec.rb
|
117
|
+
- spec/spec_helper.rb
|
118
|
+
homepage: ''
|
119
|
+
licenses:
|
120
|
+
- MIT
|
121
|
+
metadata: {}
|
122
|
+
post_install_message:
|
123
|
+
rdoc_options: []
|
124
|
+
require_paths:
|
125
|
+
- lib
|
126
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
127
|
+
requirements:
|
128
|
+
- - '>='
|
129
|
+
- !ruby/object:Gem::Version
|
130
|
+
version: '0'
|
131
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
132
|
+
requirements:
|
133
|
+
- - '>='
|
134
|
+
- !ruby/object:Gem::Version
|
135
|
+
version: '0'
|
136
|
+
requirements: []
|
137
|
+
rubyforge_project:
|
138
|
+
rubygems_version: 2.1.9
|
139
|
+
signing_key:
|
140
|
+
specification_version: 4
|
141
|
+
summary: Convenient work with GoodData's Datawarehouse (ADS)
|
142
|
+
test_files:
|
143
|
+
- spec/data/bike.csv
|
144
|
+
- spec/datawarehouse_spec.rb
|
145
|
+
- spec/spec_helper.rb
|