redata 1.0.0 → 1.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +8 -8
- data/bin/adjust +1 -1
- data/lib/redata/config.rb +4 -0
- data/lib/redata/database.rb +5 -5
- data/lib/redata/log.rb +14 -4
- data/lib/redata/parser.rb +1 -1
- data/lib/redata/relation.rb +1 -1
- data/lib/redata/requires.rb +1 -0
- data/lib/redata/ssh.rb +13 -3
- data/lib/redata/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: ef52bd600d45561f6a13e060d1417a00545279ef
|
4
|
+
data.tar.gz: c3c9959096211dae0b9f5ea8379c144f18dca51e
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 463e9ed7bbc5a45d85c9fe1bc45c9811cd7f2ee3d6ab4d0750a274552c48efc50942eeb3a32607e9d388968a0d70301dfeb5165747f9d63141a9e352a98bd382
|
7
|
+
data.tar.gz: 6bb1724724771231af491b76cbc13fd52ef29aabe71bdb5d6812f25f04700bf5317f6bda31ffb1147e4100fec2ad252dab4ec0e1507bf6872b7063699fe793cc
|
data/README.md
CHANGED
@@ -47,7 +47,7 @@ slack_bot: # this setting will be used for slack notice push
|
|
47
47
|
channel: slack_channel
|
48
48
|
```
|
49
49
|
|
50
|
-
+ config `config/
|
50
|
+
+ config `config/red_access.yml` for development and production environment in redshift database
|
51
51
|
eg.
|
52
52
|
|
53
53
|
```YAML
|
@@ -75,14 +75,14 @@ Redata::Task.schema.config do
|
|
75
75
|
# Example of declaring a global table
|
76
76
|
table 'table_name'
|
77
77
|
# This declaration means
|
78
|
-
# query file:
|
78
|
+
# query file: red_query/sources/table_name.red.sql
|
79
79
|
# redshift table: table_name
|
80
80
|
# key used in command line: table_name
|
81
81
|
|
82
82
|
# Example of declaring a global table with customizing options
|
83
83
|
table 'new_table_name', :dir => 'dir', :file => 'query_file', :as => :alias
|
84
84
|
# This declaration means
|
85
|
-
# query file:
|
85
|
+
# query file: red_query/sources/dir/query_file.red.sql
|
86
86
|
# redshift table: new_table_name
|
87
87
|
# key used in command line: alias
|
88
88
|
|
@@ -94,13 +94,13 @@ Redata::Task.schema.config do
|
|
94
94
|
category :test_category do
|
95
95
|
table 'test_table'
|
96
96
|
# This declaration means
|
97
|
-
# query file:
|
97
|
+
# query file: red_query/sources/test_category/test_table.red.sql
|
98
98
|
# redshift table: test_category_test_table
|
99
99
|
# key used in command line: test_category_test_table
|
100
100
|
|
101
101
|
table 'test_table_oth', :dir => 'dir', :file => 'query_file_oth', :as => :alias_oth
|
102
102
|
# This declaration means
|
103
|
-
# query file:
|
103
|
+
# query file: red_query/sources/dir/query_file_oth.red.sql
|
104
104
|
# redshift table: test_category_test_table
|
105
105
|
# key used in command line: test_category_alias_oth
|
106
106
|
|
@@ -165,7 +165,7 @@ Usage: `redata [-options] [action] [object key] {stage}`
|
|
165
165
|
- checkout --> export data in table/view of redshift into S3
|
166
166
|
- deploy --> deploy data from S3 to local db or file
|
167
167
|
+ object key --> object declared in `config/relation.rb` will be create/delete/checkout/deploy
|
168
|
-
+ stage --> when injecting data into mysql, there may be several stage declared in `config/
|
168
|
+
+ stage --> when injecting data into mysql, there may be several stage declared in `config/red_access.yml{:deploy}` for same database, this could choose which stage to use.
|
169
169
|
+ options
|
170
170
|
- -dir --> project directory, both absolute path and realtive path will be okay. default is current directory.
|
171
171
|
- -e --> environment: `production`, `development`, etc.
|
@@ -178,8 +178,8 @@ Usage: `redata [-options] [action] [object key] {stage}`
|
|
178
178
|
|
179
179
|
Use adjust when you just want to run a query file without declaring in `config/relations.rb`
|
180
180
|
Usage: `adjust [-options] [database] [query file] {platform}`
|
181
|
-
+ database --> `redshift` or database declared in `config/
|
182
|
-
+ query file --> query file which will be run in `
|
181
|
+
+ database --> `redshift` or database declared in `config/red_access.yml{:deploy}`
|
182
|
+
+ query file --> query file which will be run in `red_query/adjust/`, **without extends `.red.sql`**
|
183
183
|
+ platform --> same to `redata`
|
184
184
|
+ options
|
185
185
|
- -dir --> project directory, both absolute path and realtive path will be okay. default is current directory.
|
data/bin/adjust
CHANGED
@@ -4,7 +4,7 @@ require File.expand_path '../../lib/redata', __FILE__
|
|
4
4
|
|
5
5
|
|
6
6
|
temp_config = OpenStruct.new
|
7
|
-
temp_config.query_file = Redata::RED.root.join '
|
7
|
+
temp_config.query_file = Redata::RED.root.join 'red_query', 'adjust', "#{Redata::RED.params[1]}.sql"
|
8
8
|
temp_config.tmp_file_dir = Redata::RED.root.join "tmp", "adj_#{Redata::RED.params[0]}_#{Redata::RED.params[1]}"
|
9
9
|
temp_config.tmp_exec_file = Redata::RED.root.join "tmp", "adj_#{Redata::RED.params[0]}_#{Redata::RED.params[1]}", "exec.sql"
|
10
10
|
|
data/lib/redata/config.rb
CHANGED
data/lib/redata/database.rb
CHANGED
@@ -3,8 +3,8 @@ module Redata
|
|
3
3
|
REDSHIFT_PORT = 5439
|
4
4
|
|
5
5
|
def initialize
|
6
|
-
@config = YAML.load(ERB.new(File.read(RED.root.join 'config', '
|
7
|
-
Log.error! "ERROR: Database of #{RED.env} was not declared in config/
|
6
|
+
@config = YAML.load(ERB.new(File.read(RED.root.join 'config', 'red_access.yml')).result(binding))[RED.env]
|
7
|
+
Log.error! "ERROR: Database of #{RED.env} was not declared in config/red_access.yml" unless @config
|
8
8
|
|
9
9
|
@ssh = Ssh.new
|
10
10
|
end
|
@@ -16,7 +16,7 @@ module Redata
|
|
16
16
|
@ssh.run_command "export PGPASSWORD='#{ENV['PGPASSWORD']}';#{cmd} -f ~/tmp/#{config.tmp_file_dir.basename}/exec.sql"
|
17
17
|
@ssh.remove_dir "~/tmp/#{config.tmp_file_dir.basename}"
|
18
18
|
else
|
19
|
-
|
19
|
+
@ssh.local_command "#{cmd} -f #{config.tmp_exec_file}"
|
20
20
|
end
|
21
21
|
end
|
22
22
|
|
@@ -35,7 +35,7 @@ module Redata
|
|
35
35
|
else
|
36
36
|
Log.error! "ERROR: Export config of #{config.category} was not found" unless target_config
|
37
37
|
end
|
38
|
-
|
38
|
+
@ssh.local_command cmd
|
39
39
|
end
|
40
40
|
|
41
41
|
def connect_mysql(query_file, category, stage)
|
@@ -46,7 +46,7 @@ module Redata
|
|
46
46
|
Log.error! "ERROR: Export config of #{config.category} for stage #{stage} was not found" unless target_config
|
47
47
|
|
48
48
|
cmd = "mysql #{make_mysql_cmd_params(target_config)} < #{query_file}"
|
49
|
-
|
49
|
+
@ssh.local_command cmd
|
50
50
|
end
|
51
51
|
|
52
52
|
private
|
data/lib/redata/log.rb
CHANGED
@@ -1,20 +1,30 @@
|
|
1
1
|
module Redata
|
2
2
|
class Log
|
3
3
|
def self.error!(msg)
|
4
|
-
puts
|
4
|
+
puts msg.red
|
5
|
+
self.file msg
|
5
6
|
exit 1
|
6
7
|
end
|
7
8
|
|
8
9
|
def self.warning(msg)
|
9
|
-
puts
|
10
|
+
puts msg.yellow
|
11
|
+
self.file msg
|
10
12
|
end
|
11
13
|
|
12
14
|
def self.action(msg)
|
13
|
-
puts
|
15
|
+
puts msg.cyan
|
16
|
+
self.file msg
|
14
17
|
end
|
15
18
|
|
16
19
|
def self.log(msg)
|
17
|
-
puts
|
20
|
+
puts msg
|
21
|
+
self.file msg
|
22
|
+
end
|
23
|
+
|
24
|
+
def self.file(msg)
|
25
|
+
File.open RED.log_file, 'a' do |f|
|
26
|
+
f.puts "[#{RED.current_time}] #{msg}"
|
27
|
+
end
|
18
28
|
end
|
19
29
|
end
|
20
30
|
end
|
data/lib/redata/parser.rb
CHANGED
@@ -135,7 +135,7 @@ module Redata
|
|
135
135
|
Log.error! "QUERY ERROR: syntax error for load query: #{line}" if sub.empty? || name.empty?
|
136
136
|
|
137
137
|
sub_file = in_file.parent.join "_#{sub}.red.sql"
|
138
|
-
sub_file = RED.root.join '
|
138
|
+
sub_file = RED.root.join 'red_query', 'shared', "_#{sub}.rea.sql" unless sub_file.exist?
|
139
139
|
sub_temp_tables = self.parse sub_file, out_file.dirname.join("#{name}.resql")
|
140
140
|
sub_temp_tables.each do |n|
|
141
141
|
temp_tables.push n unless temp_tables.include? n
|
data/lib/redata/relation.rb
CHANGED
data/lib/redata/requires.rb
CHANGED
data/lib/redata/ssh.rb
CHANGED
@@ -18,15 +18,25 @@ module Redata
|
|
18
18
|
end
|
19
19
|
|
20
20
|
def upload_dir(dir)
|
21
|
-
|
21
|
+
`scp -r -i #{@ssh['IdentityFile']} #{dir} #{@ssh['User']}@#{@ssh['HostName']}:~/tmp/`
|
22
22
|
end
|
23
23
|
|
24
24
|
def run_command(cmd)
|
25
|
-
|
25
|
+
out, err, status = Open3.capture3 "ssh -i #{@ssh['IdentityFile']} #{@ssh['User']}@#{@ssh['HostName']} \"#{cmd}\""
|
26
|
+
puts err unless err.empty?
|
27
|
+
Log.file out unless out.empty?
|
28
|
+
Log.file err unless err.empty?
|
29
|
+
end
|
30
|
+
|
31
|
+
def local_command(cmd)
|
32
|
+
out, err, status = Open3.capture3 cmd
|
33
|
+
puts err unless err.empty?
|
34
|
+
Log.file out unless out.empty?
|
35
|
+
Log.file err unless err.empty?
|
26
36
|
end
|
27
37
|
|
28
38
|
def remove_dir(dir)
|
29
|
-
|
39
|
+
`ssh -i #{@ssh['IdentityFile']} #{@ssh['User']}@#{@ssh['HostName']} \"rm -rf #{dir}\"`
|
30
40
|
end
|
31
41
|
end
|
32
42
|
end
|
data/lib/redata/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: redata
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- goshan
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-02-
|
11
|
+
date: 2017-02-24 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|