redata 1.2.2 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: beb1e936a4f2b0ce83ef9c41d9ec3c4f69425f50
-  data.tar.gz: b323d2d3268c53250b40b621dfed6b0138721220
+  metadata.gz: 364c1b4ae5b02af7e3299a2319e712b60013814c
+  data.tar.gz: dd1b500748255322c064f7fd406499bc5968812d
 SHA512:
-  metadata.gz: 178a99c06a6d7587f9591e84748d98bf2cbea92406264c09be0ce432cab140bd10d089b7a7f5475dd07051882ff11e2199af7ebc6bad3622630a00917c494be2
-  data.tar.gz: becb4560e1c6e5b069eb4d804f8941c9173a35b56293be22d0345652b05e29480f8c3e1fc1c9acf5172fef3ec5561cccbf47cb9b0c10f1e19e32df0bb3bf610b
+  metadata.gz: c562bc89b09bcba1e9c54a2255725d30653e0c1e90a65695d5b9fcc81eaa1047185ba66bc2df33b54698e5642e20fdf5bb9a6a9cf8e219a2c8a6719d7ba8ec62
+  data.tar.gz: 0da7d784be70751f2e225d8a484bc32a1b06bb99c3841ffec68502515e57b3c4c962f775d9674ac151941e3f33b5e5266b7aa2b4efd98dcbcd1cea3aeccbde0a

data/README.md

@@ -19,6 +19,10 @@ Or install it yourself as:
 
     $ gem install redata
 
+And then you should generate some config files by
+
+	$ bundle exec rails generate redata:config
+
 ## Usage
 
 ### Config
@@ -58,13 +62,13 @@ development:
   database: dev
   deploy: # target platform db(mysql) which export data to
     app:  # category name, using database
-	  pro:  # stage name(you can still declare under category absolutely)
+      pro:  # stage name(you can still declare under category absolutely)
         username: root
         password: ''
         host: localhost
         database: app
     file:  # another category, using local file
-	  local_dir: '~/data'
+      local_dir: '~/data'
 ```
 
 + config `config/relations.rb` for data object in redshift and exporting process to mysql  
@@ -75,14 +79,14 @@ Redata::Task.schema.config do
   # Example of declaring a global table
   table 'table_name'
   # This declaration means
-  #   query file: red_query/sources/table_name.red.sql
+  #   query file: query/sources/table_name.red.sql
   #   redshift table: table_name
   #   key used in command line: table_name
 
   # Example of declaring a global table with customizing options
   table 'new_table_name', :dir => 'dir', :file => 'query_file', :as => :alias
   # This declaration means
-  #   query file: red_query/sources/dir/query_file.red.sql
+  #   query file: query/sources/dir/query_file.red.sql
   #   redshift table: new_table_name
   #   key used in command line: alias
 
@@ -94,13 +98,13 @@ Redata::Task.schema.config do
   category :test_category do
     table 'test_table'
     # This declaration means
-    #   query file: red_query/sources/test_category/test_table.red.sql
+    #   query file: query/sources/test_category/test_table.red.sql
     #   redshift table: test_category_test_table
     #   key used in command line: test_category_test_table
 
     table 'test_table_oth', :dir => 'dir', :file => 'query_file_oth', :as => :alias_oth
     # This declaration means
-    #   query file: red_query/sources/dir/query_file_oth.red.sql
+    #   query file: query/sources/dir/query_file_oth.red.sql
     #   redshift table: test_category_test_table
     #   key used in command line: test_category_alias_oth
 
@@ -179,7 +183,7 @@ Usage: `redata [-options] [action] [object key] {stage}`
 Use adjust when you just want to run a query file without declaring in `config/relations.rb`  
 Usage: `adjust [-options] [database] [query file] {platform}`
 + database   --> `redshift` or database declared in `config/red_access.yml{:deploy}`
-+ query file --> query file which will be run in `red_query/adjust/`, **without extends `.red.sql`**
++ query file --> query file which will be run in `query/adjust/`, **without extends `.red.sql`**
 + platform   --> same to `redata`
 + options
   - -dir --> project directory, both absolute path and realtive path will be okay. default is current directory.

data/bin/adjust

@@ -9,7 +9,7 @@ unless Redata::RED.params[0]
 end
 
 temp_config = OpenStruct.new
-temp_config.query_file = Redata::RED.root.join 'red_query', 'adjust', "#{Redata::RED.params[1]}.sql"
+temp_config.query_file = Redata::RED.root.join 'query', 'adjust', "#{Redata::RED.params[1]}.sql"
 temp_config.tmp_file_dir = Redata::RED.root.join "tmp", "adj_#{Redata::RED.params[0]}_#{Redata::RED.params[1]}"
 temp_config.tmp_exec_file = Redata::RED.root.join "tmp", "adj_#{Redata::RED.params[0]}_#{Redata::RED.params[1]}", "exec.sql"
 

data/lib/generators/redata/config_generator.rb

@@ -0,0 +1,16 @@
+require 'rails/generators/base'
+
+module Redata
+  class ConfigGenerator < Rails::Generators::Base
+    source_root File.expand_path('../../templates', __FILE__)
+    desc 'Redata Config'
+
+    def config_steps
+      copy_file 'source_query_example.red.sql', 'app/query/sources/sources_query_example.red.sql'
+      copy_file 'adjust_query_example.red.sql', 'app/query/adjust/adjust_query_example.red.sql'
+      copy_file 'redata.yml', 'config/redata.yml'
+      copy_file 'red_access.yml', 'config/red_access.yml'
+      copy_file 'relations.rb', 'config/relations.rb'
+    end
+  end
+end

data/lib/generators/templates/adjust_query_example.red.sql

@@ -0,0 +1,3 @@
+-- you can use all syntax same as queries in sources
+
+UPDATE [table] SET [attr] = [value] WHERE [key] = [kw];

data/lib/generators/templates/red_access.yml

@@ -0,0 +1,16 @@
+# Access configuration of redshift and local database
+#
+development:
+  host: localhost
+  username: user
+  password: ''
+  database: dev
+  deploy: # target platform db(mysql) which export data to
+    app:  # category name, using database
+      pro:  # stage name(you can still declare under category absolutely)
+        username: root
+        password: ''
+        host: localhost
+        database: app
+    file:  # another category, using local file
+      local_dir: '~/data'

data/lib/generators/templates/redata.yml

@@ -0,0 +1,21 @@
+# Basic configuration of redata
+#
+create_interval:  # default date for create mode
+  start_time: "2016-04-04"
+  end_time: 2  # days ago
+append_interval:  # date fetching interval for append mode
+  start: 3  # days ago
+  end: 2  # days ago
+timezone: "Asia/Tokyo"
+keep_tmp: true    # or false. whether keep temp query file in ./tmp after finished query
+s3:
+  bucket: bucket_name
+  aws_access_key_id: key_id
+  aws_secret_access_key: key_secret
+ssh:  # this setting will be used in ssh mode when you access private database
+  HostName: gateway_host
+  IdentityFile: ~/.ssh/key.pem
+  User: username
+slack_bot:  # this setting will be used for slack notice push
+  token: bot_token
+  channel: slack_channel

data/lib/generators/templates/relations.rb

@@ -0,0 +1,39 @@
+Redata::Task.schema.config do
+  # Example of declaring a global table
+  table 'source_query_example'
+  # This declaration means
+  #   query file: query/sources/source_query_example.red.sql
+  #   redshift table: source_query_example
+  #   key used in command line: source_query_example
+
+  # Example of declaring a global table with customizing options
+  # table 'new_table_name', :dir => 'dir', :file => 'query_file', :as => :alias
+  # This declaration means
+  #   query file: red_query/sources/dir/query_file.red.sql
+  #   redshift table: new_table_name
+  #   key used in command line: alias
+
+  # view is same to table but will still be created in append_mode
+  # view 'view_name'
+  # view 'new_view_name', :dir => 'dir', :file => 'query_file_oth', :as => :alias_oth
+
+  # Example of declaring with category
+  # category :test_category do
+  #   table 'test_table'
+  #   This declaration means
+  #     query file: red_query/sources/test_category/test_table.red.sql
+  #     redshift table: test_category_test_table
+  #     key used in command line: test_category_test_table
+
+  #   table 'test_table_oth', :dir => 'dir', :file => 'query_file_oth', :as => :alias_oth
+  #   This declaration means
+  #     query file: red_query/sources/dir/query_file_oth.red.sql
+  #     redshift table: test_category_test_table
+  #     key used in command line: test_category_alias_oth
+
+  #   view is same to table without appending update type
+  #   view 'test_view'
+  #   view 'test_view_oth', :dir => 'dir', :file => 'query_file_oth', :as => :alias_view_oth
+  # end
+
+end

data/lib/generators/templates/source_query_example.red.sql

@@ -0,0 +1,28 @@
+-- query file in data/sources/*.red.sql
+
+#load 'sub_query_a' --> :a  -- include a sub query as object a from _sub_query_a.red.sql in same folder
+#load 'sub_query_b' --> :b
+
+
+-- use can use if logic to control whether run part of a query
+-- 'endif' could stop one or many continuous if logic above. (use if which is from second just like 'else if')
+-- TIPS: we have not supported 'else if', 'else' syntax and nested if logic
+[if var is 'value1']
+SELECT a.col1, a.col2, b.col1, b.col2, b.col3
+[if var is 'value1']
+SELECT a.col3, b.col4
+[endif]
+FROM {a}  -- use object a included from sub query file '_sub_query_a.sql'
+JOIN {b} ON b.col1 = a.col1
+-- For [start_time] and [end_time], there are 3 options.
+-- use command params when set
+-- in append mode, use [append_interval][start_time] or [append_interval][end_time] (See config/redata.yml).
+-- in create mode, use [create_interval][start_time] or [create_interval][end_time] (See config/redata.yml).
+WHERE a.col1 >= [start_time]
+AND a.col1 < [end_time]
+-- some params getting from command input such as `-param_from_command param_value`
+AND a.col2 = [param_from_command]
+-- current time in setted timezone will be used (About timezon, also see config/redata.yml)
+AND b.col2 <= [current_time]
+-- x days before today, x will be a integer
+AND b.col3 >= [x days ago]

data/lib/redata/parser.rb

@@ -143,7 +143,7 @@ module Redata
 							RED.locals[key] = val
 						end
 						sub_file = in_file.parent.join "_#{sub}.red.sql"
-						sub_file = RED.root.join 'red_query', 'shared', "_#{sub}.rea.sql" unless sub_file.exist?
+						sub_file = RED.root.join 'query', 'shared', "_#{sub}.rea.sql" unless sub_file.exist?
 						sub_temp_tables = self.parse sub_file, out_file.dirname.join("#{name}.resql")
 						sub_temp_tables.each do |n|
 							temp_tables.push n unless temp_tables.include? n

data/lib/redata/relation.rb

@@ -18,7 +18,7 @@ module Redata
 		end
 
 		def query_file
-			query_file = RED.root.join 'red_query', 'sources'
+			query_file = RED.root.join 'query', 'sources'
 			query_file = query_file.join @dir if @dir
 			query_file = query_file.join "#{@file}.red.sql"
 			query_file

data/lib/redata/version.rb

@@ -1,3 +1,3 @@
 module Redata
-  VERSION = "1.2.2"
+  VERSION = "1.3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redata
 version: !ruby/object:Gem::Version
-  version: 1.2.2
+  version: 1.3.0
 platform: ruby
 authors:
 - goshan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-03-22 00:00:00.000000000 Z
+date: 2017-03-31 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -128,6 +128,12 @@ files:
 - bin/notice
 - bin/redata
 - bin/setup
+- lib/generators/redata/config_generator.rb
+- lib/generators/templates/adjust_query_example.red.sql
+- lib/generators/templates/red_access.yml
+- lib/generators/templates/redata.yml
+- lib/generators/templates/relations.rb
+- lib/generators/templates/source_query_example.red.sql
 - lib/redata.rb
 - lib/redata/bucket.rb
 - lib/redata/config.rb