redata 1.2.2 → 1.3.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +11 -7
- data/bin/adjust +1 -1
- data/lib/generators/redata/config_generator.rb +16 -0
- data/lib/generators/templates/adjust_query_example.red.sql +3 -0
- data/lib/generators/templates/red_access.yml +16 -0
- data/lib/generators/templates/redata.yml +21 -0
- data/lib/generators/templates/relations.rb +39 -0
- data/lib/generators/templates/source_query_example.red.sql +28 -0
- data/lib/redata/parser.rb +1 -1
- data/lib/redata/relation.rb +1 -1
- data/lib/redata/version.rb +1 -1
- metadata +8 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 364c1b4ae5b02af7e3299a2319e712b60013814c
|
4
|
+
data.tar.gz: dd1b500748255322c064f7fd406499bc5968812d
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: c562bc89b09bcba1e9c54a2255725d30653e0c1e90a65695d5b9fcc81eaa1047185ba66bc2df33b54698e5642e20fdf5bb9a6a9cf8e219a2c8a6719d7ba8ec62
|
7
|
+
data.tar.gz: 0da7d784be70751f2e225d8a484bc32a1b06bb99c3841ffec68502515e57b3c4c962f775d9674ac151941e3f33b5e5266b7aa2b4efd98dcbcd1cea3aeccbde0a
|
data/README.md
CHANGED
@@ -19,6 +19,10 @@ Or install it yourself as:
|
|
19
19
|
|
20
20
|
$ gem install redata
|
21
21
|
|
22
|
+
And then you should generate some config files by
|
23
|
+
|
24
|
+
$ bundle exec rails generate redata:config
|
25
|
+
|
22
26
|
## Usage
|
23
27
|
|
24
28
|
### Config
|
@@ -58,13 +62,13 @@ development:
|
|
58
62
|
database: dev
|
59
63
|
deploy: # target platform db(mysql) which export data to
|
60
64
|
app: # category name, using database
|
61
|
-
|
65
|
+
pro: # stage name(you can still declare under category absolutely)
|
62
66
|
username: root
|
63
67
|
password: ''
|
64
68
|
host: localhost
|
65
69
|
database: app
|
66
70
|
file: # another category, using local file
|
67
|
-
|
71
|
+
local_dir: '~/data'
|
68
72
|
```
|
69
73
|
|
70
74
|
+ config `config/relations.rb` for data object in redshift and exporting process to mysql
|
@@ -75,14 +79,14 @@ Redata::Task.schema.config do
|
|
75
79
|
# Example of declaring a global table
|
76
80
|
table 'table_name'
|
77
81
|
# This declaration means
|
78
|
-
# query file:
|
82
|
+
# query file: query/sources/table_name.red.sql
|
79
83
|
# redshift table: table_name
|
80
84
|
# key used in command line: table_name
|
81
85
|
|
82
86
|
# Example of declaring a global table with customizing options
|
83
87
|
table 'new_table_name', :dir => 'dir', :file => 'query_file', :as => :alias
|
84
88
|
# This declaration means
|
85
|
-
# query file:
|
89
|
+
# query file: query/sources/dir/query_file.red.sql
|
86
90
|
# redshift table: new_table_name
|
87
91
|
# key used in command line: alias
|
88
92
|
|
@@ -94,13 +98,13 @@ Redata::Task.schema.config do
|
|
94
98
|
category :test_category do
|
95
99
|
table 'test_table'
|
96
100
|
# This declaration means
|
97
|
-
# query file:
|
101
|
+
# query file: query/sources/test_category/test_table.red.sql
|
98
102
|
# redshift table: test_category_test_table
|
99
103
|
# key used in command line: test_category_test_table
|
100
104
|
|
101
105
|
table 'test_table_oth', :dir => 'dir', :file => 'query_file_oth', :as => :alias_oth
|
102
106
|
# This declaration means
|
103
|
-
# query file:
|
107
|
+
# query file: query/sources/dir/query_file_oth.red.sql
|
104
108
|
# redshift table: test_category_test_table
|
105
109
|
# key used in command line: test_category_alias_oth
|
106
110
|
|
@@ -179,7 +183,7 @@ Usage: `redata [-options] [action] [object key] {stage}`
|
|
179
183
|
Use adjust when you just want to run a query file without declaring in `config/relations.rb`
|
180
184
|
Usage: `adjust [-options] [database] [query file] {platform}`
|
181
185
|
+ database --> `redshift` or database declared in `config/red_access.yml{:deploy}`
|
182
|
-
+ query file --> query file which will be run in `
|
186
|
+
+ query file --> query file which will be run in `query/adjust/`, **without extends `.red.sql`**
|
183
187
|
+ platform --> same to `redata`
|
184
188
|
+ options
|
185
189
|
- -dir --> project directory, both absolute path and realtive path will be okay. default is current directory.
|
data/bin/adjust
CHANGED
@@ -9,7 +9,7 @@ unless Redata::RED.params[0]
|
|
9
9
|
end
|
10
10
|
|
11
11
|
temp_config = OpenStruct.new
|
12
|
-
temp_config.query_file = Redata::RED.root.join '
|
12
|
+
temp_config.query_file = Redata::RED.root.join 'query', 'adjust', "#{Redata::RED.params[1]}.sql"
|
13
13
|
temp_config.tmp_file_dir = Redata::RED.root.join "tmp", "adj_#{Redata::RED.params[0]}_#{Redata::RED.params[1]}"
|
14
14
|
temp_config.tmp_exec_file = Redata::RED.root.join "tmp", "adj_#{Redata::RED.params[0]}_#{Redata::RED.params[1]}", "exec.sql"
|
15
15
|
|
@@ -0,0 +1,16 @@
|
|
1
|
+
require 'rails/generators/base'
|
2
|
+
|
3
|
+
module Redata
|
4
|
+
class ConfigGenerator < Rails::Generators::Base
|
5
|
+
source_root File.expand_path('../../templates', __FILE__)
|
6
|
+
desc 'Redata Config'
|
7
|
+
|
8
|
+
def config_steps
|
9
|
+
copy_file 'source_query_example.red.sql', 'app/query/sources/sources_query_example.red.sql'
|
10
|
+
copy_file 'adjust_query_example.red.sql', 'app/query/adjust/adjust_query_example.red.sql'
|
11
|
+
copy_file 'redata.yml', 'config/redata.yml'
|
12
|
+
copy_file 'red_access.yml', 'config/red_access.yml'
|
13
|
+
copy_file 'relations.rb', 'config/relations.rb'
|
14
|
+
end
|
15
|
+
end
|
16
|
+
end
|
@@ -0,0 +1,16 @@
|
|
1
|
+
# Access configuration of redshift and local database
|
2
|
+
#
|
3
|
+
development:
|
4
|
+
host: localhost
|
5
|
+
username: user
|
6
|
+
password: ''
|
7
|
+
database: dev
|
8
|
+
deploy: # target platform db(mysql) which export data to
|
9
|
+
app: # category name, using database
|
10
|
+
pro: # stage name(you can still declare under category absolutely)
|
11
|
+
username: root
|
12
|
+
password: ''
|
13
|
+
host: localhost
|
14
|
+
database: app
|
15
|
+
file: # another category, using local file
|
16
|
+
local_dir: '~/data'
|
@@ -0,0 +1,21 @@
|
|
1
|
+
# Basic configuration of redata
|
2
|
+
#
|
3
|
+
create_interval: # default date for create mode
|
4
|
+
start_time: "2016-04-04"
|
5
|
+
end_time: 2 # days ago
|
6
|
+
append_interval: # date fetching interval for append mode
|
7
|
+
start: 3 # days ago
|
8
|
+
end: 2 # days ago
|
9
|
+
timezone: "Asia/Tokyo"
|
10
|
+
keep_tmp: true # or false. whether keep temp query file in ./tmp after finished query
|
11
|
+
s3:
|
12
|
+
bucket: bucket_name
|
13
|
+
aws_access_key_id: key_id
|
14
|
+
aws_secret_access_key: key_secret
|
15
|
+
ssh: # this setting will be used in ssh mode when you access private database
|
16
|
+
HostName: gateway_host
|
17
|
+
IdentityFile: ~/.ssh/key.pem
|
18
|
+
User: username
|
19
|
+
slack_bot: # this setting will be used for slack notice push
|
20
|
+
token: bot_token
|
21
|
+
channel: slack_channel
|
@@ -0,0 +1,39 @@
|
|
1
|
+
Redata::Task.schema.config do
|
2
|
+
# Example of declaring a global table
|
3
|
+
table 'source_query_example'
|
4
|
+
# This declaration means
|
5
|
+
# query file: query/sources/source_query_example.red.sql
|
6
|
+
# redshift table: source_query_example
|
7
|
+
# key used in command line: source_query_example
|
8
|
+
|
9
|
+
# Example of declaring a global table with customizing options
|
10
|
+
# table 'new_table_name', :dir => 'dir', :file => 'query_file', :as => :alias
|
11
|
+
# This declaration means
|
12
|
+
# query file: red_query/sources/dir/query_file.red.sql
|
13
|
+
# redshift table: new_table_name
|
14
|
+
# key used in command line: alias
|
15
|
+
|
16
|
+
# view is same to table but will still be created in append_mode
|
17
|
+
# view 'view_name'
|
18
|
+
# view 'new_view_name', :dir => 'dir', :file => 'query_file_oth', :as => :alias_oth
|
19
|
+
|
20
|
+
# Example of declaring with category
|
21
|
+
# category :test_category do
|
22
|
+
# table 'test_table'
|
23
|
+
# This declaration means
|
24
|
+
# query file: red_query/sources/test_category/test_table.red.sql
|
25
|
+
# redshift table: test_category_test_table
|
26
|
+
# key used in command line: test_category_test_table
|
27
|
+
|
28
|
+
# table 'test_table_oth', :dir => 'dir', :file => 'query_file_oth', :as => :alias_oth
|
29
|
+
# This declaration means
|
30
|
+
# query file: red_query/sources/dir/query_file_oth.red.sql
|
31
|
+
# redshift table: test_category_test_table
|
32
|
+
# key used in command line: test_category_alias_oth
|
33
|
+
|
34
|
+
# view is same to table without appending update type
|
35
|
+
# view 'test_view'
|
36
|
+
# view 'test_view_oth', :dir => 'dir', :file => 'query_file_oth', :as => :alias_view_oth
|
37
|
+
# end
|
38
|
+
|
39
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
-- query file in data/sources/*.red.sql
|
2
|
+
|
3
|
+
#load 'sub_query_a' --> :a -- include a sub query as object a from _sub_query_a.red.sql in same folder
|
4
|
+
#load 'sub_query_b' --> :b
|
5
|
+
|
6
|
+
|
7
|
+
-- use can use if logic to control whether run part of a query
|
8
|
+
-- 'endif' could stop one or many continuous if logic above. (use if which is from second just like 'else if')
|
9
|
+
-- TIPS: we have not supported 'else if', 'else' syntax and nested if logic
|
10
|
+
[if var is 'value1']
|
11
|
+
SELECT a.col1, a.col2, b.col1, b.col2, b.col3
|
12
|
+
[if var is 'value1']
|
13
|
+
SELECT a.col3, b.col4
|
14
|
+
[endif]
|
15
|
+
FROM {a} -- use object a included from sub query file '_sub_query_a.sql'
|
16
|
+
JOIN {b} ON b.col1 = a.col1
|
17
|
+
-- For [start_time] and [end_time], there are 3 options.
|
18
|
+
-- use command params when set
|
19
|
+
-- in append mode, use [append_interval][start_time] or [append_interval][end_time] (See config/redata.yml).
|
20
|
+
-- in create mode, use [create_interval][start_time] or [create_interval][end_time] (See config/redata.yml).
|
21
|
+
WHERE a.col1 >= [start_time]
|
22
|
+
AND a.col1 < [end_time]
|
23
|
+
-- some params getting from command input such as `-param_from_command param_value`
|
24
|
+
AND a.col2 = [param_from_command]
|
25
|
+
-- current time in setted timezone will be used (About timezon, also see config/redata.yml)
|
26
|
+
AND b.col2 <= [current_time]
|
27
|
+
-- x days before today, x will be a integer
|
28
|
+
AND b.col3 >= [x days ago]
|
data/lib/redata/parser.rb
CHANGED
@@ -143,7 +143,7 @@ module Redata
|
|
143
143
|
RED.locals[key] = val
|
144
144
|
end
|
145
145
|
sub_file = in_file.parent.join "_#{sub}.red.sql"
|
146
|
-
sub_file = RED.root.join '
|
146
|
+
sub_file = RED.root.join 'query', 'shared', "_#{sub}.rea.sql" unless sub_file.exist?
|
147
147
|
sub_temp_tables = self.parse sub_file, out_file.dirname.join("#{name}.resql")
|
148
148
|
sub_temp_tables.each do |n|
|
149
149
|
temp_tables.push n unless temp_tables.include? n
|
data/lib/redata/relation.rb
CHANGED
data/lib/redata/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: redata
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.3.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- goshan
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-03-
|
11
|
+
date: 2017-03-31 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: bundler
|
@@ -128,6 +128,12 @@ files:
|
|
128
128
|
- bin/notice
|
129
129
|
- bin/redata
|
130
130
|
- bin/setup
|
131
|
+
- lib/generators/redata/config_generator.rb
|
132
|
+
- lib/generators/templates/adjust_query_example.red.sql
|
133
|
+
- lib/generators/templates/red_access.yml
|
134
|
+
- lib/generators/templates/redata.yml
|
135
|
+
- lib/generators/templates/relations.rb
|
136
|
+
- lib/generators/templates/source_query_example.red.sql
|
131
137
|
- lib/redata.rb
|
132
138
|
- lib/redata/bucket.rb
|
133
139
|
- lib/redata/config.rb
|