redata 0.1.0 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 58c04577fb55ac865dda405347811acf421dac3e
-  data.tar.gz: a1a33fb66a93f198aca56731417c2a2d90b65192
+  metadata.gz: ba347eb745baeb8cf5ffee03842482bfcc2b98d7
+  data.tar.gz: cb86f147ae139d3e87af8afa6e74a823abe02c02
 SHA512:
-  metadata.gz: 199b5da361e782c96ba4dbb1c5e465ca3fd7e9a8f4471776efb0915408ec9f43428aadfda25f5037e883c4798bbec4633c2e0aa7ddb76d59daafc3e8c0ed56dc
-  data.tar.gz: 551db6a70bbcdfd29a39aed84d69258d883afcbede934bc5f7fc08ed165b8c3978671f2cba2e8d26ee3bc5a6a8349cc4cde86028824241b01dfce1fa6e6dd092
+  metadata.gz: 47cabaf3d9fcab6058a587c823eeee1a6bc39a33051b7675e176dcd7b812e33678ecd89e2f4b4463747813ade4f287947119623b7ec19b45cbf9dbf316a87dc4
+  data.tar.gz: 13991f7d3daa93f15f31bd1e6654147098c98f36f7d1eb19f2003f58792ede7c9b806be33571593d138db5f2425ef9e63444103b8ff9063d98778e14bfdd9dc7

data/README.md

@@ -26,22 +26,25 @@ Or install it yourself as:
 + config `config/redata.yml` for general setting
 
 ```YAML
-start_date: "2016-04-04"  # default data start date
+create_interval:  # default date for create mode
+  start_time: "2016-04-04"
+  end_time: 2  # days ago
+append_interval:  # date fetching interval for append mode
+  start: 3  # days ago
+  end: 2  # days ago
 timezone: "Asia/Tokyo"
+keep_tmp: true    # or false. whether keep temp query file in ./tmp after finished query
 s3:
-  aws_access_key_id: {key_id}
-  aws_secret_access_key: {key_secret}
-  region: {s3_region}
-  bucket:
-	production: {bucket_name}
-	development: {bucket_name}
+  bucket: bucket_name
+  aws_access_key_id: key_id
+  aws_secret_access_key: key_secret
 ssh:  # this setting will be used in ssh mode when you access private database
-  HostName: {gateway_host}
-  IdentityFile: {~/.ssh/key.pem}
-  User: {username}
+  HostName: gateway_host
+  IdentityFile: ~/.ssh/key.pem
+  User: username
 slack_bot:  # this setting will be used for slack notice push
-  token: {bot_token}
-  channel: {slack_channel}
+  token: bot_token
+  channel: slack_channel
 ```
 
 + config `config/database.yml` for development and production environment in redshift database  
@@ -53,12 +56,15 @@ development:
   username: user
   password: ''
   database: dev
-  export: # target platform db(mysql) which export data to
-    app:  # platform name
-      username: root
-      password: ''
-      host: localhost
-      database: app
+  deploy: # target platform db(mysql) which export data to
+    app:  # category name, using database
+	  pro:  # stage name(you can still declare under category absolutely)
+        username: root
+        password: ''
+        host: localhost
+        database: app
+    file:  # another category, using local file
+	  local_dir: '~/data'
 ```
 
 + config `config/relations.rb` for data object in redshift and exporting process to mysql  
@@ -69,60 +75,38 @@ Redata::Task.schema.config do
   # Example of declaring a global table
   table 'table_name'
   # This declaration means
-  #   query file: database/sources/table_name.sql
+  #   query file: database/sources/table_name.red.sql
   #   redshift table: table_name
-  #   update type: renewal, delete and re-create when update
   #   key used in command line: table_name
-  
+
   # Example of declaring a global table with customizing options
-  table 'new_table_name', :dir => 'dir', :file => 'query_file', :update => :append, :as => :alias
+  table 'new_table_name', :dir => 'dir', :file => 'query_file', :as => :alias
   # This declaration means
-  #   query file: database/sources/dir/query_file.sql
+  #   query file: database/sources/dir/query_file.red.sql
   #   redshift table: new_table_name
-  #   update type: append, only appending to existing table
   #   key used in command line: alias
-  
-  # view is same to table but the update type only has renewal mode
-  table 'view_name'
-  table 'new_view_name', :dir => 'dir', :file => 'query_file_oth', :as => :alias_oth
-  
+
+  # view is same to table but will still be created in append_mode
+  view 'view_name'
+  view 'new_view_name', :dir => 'dir', :file => 'query_file_oth', :as => :alias_oth
+
   # Example of declaring with category
   category :test_category do
     table 'test_table'
     # This declaration means
-    #   query file: database/sources/test_category/test_table.sql
+    #   query file: database/sources/test_category/test_table.red.sql
     #   redshift table: test_category_test_table
-    #   update type: renewal
     #   key used in command line: test_category_test_table
-    
-    table 'test_table_oth', :dir => 'dir', :file => 'query_file_oth', :update => append, :as => :alias_oth
+
+    table 'test_table_oth', :dir => 'dir', :file => 'query_file_oth', :as => :alias_oth
     # This declaration means
-    #   query file: database/sources/dir/query_file_oth.sql
+    #   query file: database/sources/dir/query_file_oth.red.sql
     #   redshift table: test_category_test_table
-    #   update type: append
     #   key used in command line: test_category_alias_oth
-    
+
     # view is same to table without appending update type
     view 'test_view'
     view 'test_view_oth', :dir => 'dir', :file => 'query_file_oth', :as => :alias_view_oth
-    
-    #Example of convertor declaration
-    export 'test_export'
-    # This declaration means
-    #   convertor file: database/convertors/test_category/test_export.conv
-    #   target mysql database name: test_category (Also see: export config in config/database.yml{:export})
-    #   target mysql table: test_export
-    #   update type: renewal, delete all records and insert new records
-    #   key used in command line: test_category_test_export
-    
-    #Example of convertor declaration
-    export 'test_export', :dir => 'dir', :file => 'conv_file', :update => 'append', :as => 'alias_export'
-    # This declaration means
-    #   convertor file: database/convertors/dir/conv_file.conv
-    #   target mysql database name: test_category
-    #   target mysql table: test_export
-    #   update type: append, append insert new records without deleting
-    #   key used in command line: test_category_alias_export
   end
 
 end
@@ -130,50 +114,40 @@ end
 
 ### Query file
 
-Query file was used for create table of view in redshift. It is almost like PostgreSQL file but with same new feature. And you have no need to write a create table/view query, the result after running query file will used to create a new table/view, for table, if you use append mode, the result will only be append-inserted to table.  
+Query file was used for create table or view in redshift. It is almost like PostgreSQL file but with some new feature. And you have no need to write a create table/view query, the result after running query file will used to create a new table/view. For table, if you use append mode, the result will only be append-inserted to table.  
 eg.
 
 ```SQL
--- query file in data/sources/...
+-- query file in data/sources/*.red.sql
 
-#include 'sub_query_a' --> :a  -- include a sub query as object a from _sub_query_a.sql in same folder or database/shared/
-#include 'sub_query_b' --> :b
+#load 'sub_query_a' --> :a  -- include a sub query as object a from _sub_query_a.red.sql in same folder
+#load 'sub_query_b' --> :b
 
 
+-- use can use if logic to control whether run part of a query
+-- 'endif' could stop one or many continuous if logic above. (use if which is from second just like 'else if')
+-- TIPS: we have not supported 'else if', 'else' syntax and nested if logic
+[if var is 'value1']
 select a.col1, a.col2, b.col1, b.col2, b.col3
+[if var is 'value1']
+select a.col3, b.col4
+[endif]
 from {a}  -- use object a included from sub query file '_sub_query_a.sql'
 join {b} on b.col1 = a.col1
--- If in append mode and this table was setted appending update type, then start_time getting from command input such as `-start_time 2016-11-08` will be used here. When missing input this param, as default [2 days ago] will be used.
--- Or if not append mode, start_date will be used as default (Also see config/redata.yml).  set start_time when running command , if missing in command, default_start_date will be used 
+-- For [start_time] and [end_time], there are 3 options.
+-- use command params when set
+-- in append mode, use [append_interval][start_time] or [append_interval][end_time] (See config/redata.yml).
+-- in create mode, use [create_interval][start_time] or [create_interval][end_time] (See config/redata.yml).
 where a.col1 >= [start_time]
--- current time in setted timezone will be used (About timezon, also see config/redata.yml)
-and a.col1 <= [current_time]
+and a.col1 < [end_time]
 -- some params getting from command input such as `-param_from_command param_value`
 and a.col2 = [param_from_command]
+-- current time in setted timezone will be used (About timezon, also see config/redata.yml)
+and b.col2 <= [current_time]
 -- x days before today, x will be a integer
 and b.col3 >= [x days ago]
 ```
 
-### Convertor config file
-
-Convertor file was used to generate a select query to get data from redshift and unload to S3. But you have no need to wirte a unload query. If you are using append mode, only data 2 days ago will be select.   
-eg.
-
-```
-source: redshift_source_table_or_view
-columns:
-	cm_id
-	segment_type{'C' => 0, 'T' => 1, 'M1' => 2, 'M2' => 3, 'M3' => 4, 'F1' => 5, 'F2' => 6, 'F3' => 7}
-	v
-	e
-	base_ai
-	sample_num
-	grp
-```
-
-> convertor config file in `data/convertors/...`  
-> `source` means the source table in redshift  
-> `columns` means the source columns in source table  
 
 ### Command
 
@@ -184,32 +158,28 @@ There are 3 executable file in bin/
 
 #### redata
 
-Usage: `redata [-options] [action] [object key] {platform}`
+Usage: `redata [-options] [action] [object key] {stage}`
 + action
   - create   --> create a table/view or append data to table in redshift
   - delete   --> delete a table/view in redshift
   - checkout --> export data in table/view of redshift into S3
-  - inject   --> import data into mysql table from S3
-+ object key --> object will be create/delete/checkout/inject declared in `config/relation.rb`
-+ platform   --> when injecting data into mysql, there may be several platform declared in `config/database.yml{:export}` for same database, here is setting which platform to use. *If the platform here could not be found in `database.yml` or have not set platform, the default export will be used.*
+  - deploy --> deploy data from S3 to local db or file
++ object key --> object declared in `config/relation.rb` will be create/delete/checkout/deploy
++ stage --> when injecting data into mysql, there may be several stage declared in `config/database.yml{:deploy}` for same database, this could choose which stage to use.
 + options
   - -dir --> project directory, both absolute path and realtive path will be okay. default is current directory.
   - -e   --> environment: `production`, `development`, etc.
   - -f   --> force mode, use `CADCASE` when removing view or table in redshift
   - -ssh --> use ssh accessing to private database with ssh config in `config/redata.yml`
-  - -append_mode  --> use `append_mode`, the objects in relations.rb with appending update type will go to appending operation. 
-    + delete will only delete objects with renewal update type
-    + create will append-insert data after `-start_time`(set in command) or default `2 days ago` for appending update type, still create table/view for renewal type
-    + checkout will only fetch data after `-start_time` or default `2 days ago` to upload to S3, renewal type will still be uploaded all data
-    + inject will insert data to mysql without `--delete` option, renewal still delete all firstly
+  - -append  --> use `append_mode`, append new data into existing table for redshift or inject into local db without deleting. view has no append mode.
   - other options  --> some params will be used in query file when declared, such `start_time`
 
 #### adjust
 
-Use adjust when you just want to run a query file without declaring in `config/relations.rb`
+Use adjust when you just want to run a query file without declaring in `config/relations.rb`  
 Usage: `adjust [-options] [database] [query file] {platform}`
-+ database   --> `redshift` or database declared in `config/database.yml{export}`
-+ query file --> query file which will be run in `database/adjust/`, **without extends `.sql`**
++ database   --> `redshift` or database declared in `config/database.yml{:deploy}`
++ query file --> query file which will be run in `database/adjust/`, **without extends `.red.sql`**
 + platform   --> same to `redata`
 + options
   - -dir --> project directory, both absolute path and realtive path will be okay. default is current directory.
@@ -220,9 +190,9 @@ Usage: `adjust [-options] [database] [query file] {platform}`
 #### notice
 
 Usage: `notice [-options] [action]`
-+ action: currently, there is only `update` action which means send 'finish updating' message to slack
-+ options
-  - -e   --> environment: `production`, `development`, etc. **Only production could send notice**
++ action
+  - log      --> send a message to slack with a log file
+  - mention  --> send a message to slack with mention someone
 
 ## Contributing
 
@@ -231,5 +201,25 @@ Bug reports and pull requests are welcome on GitHub at https://github.com/goshan
 
 ## License
 
-The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+Copyright 2013, Han Qiu(goshan), All rights reserved.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
 

data/bin/adjust

@@ -3,15 +3,20 @@
 require File.expand_path '../../lib/redata', __FILE__
 
 
-query_file = Redata::RED.root.join 'database', 'adjust', "#{Redata::RED.params[1]}.sql"
-tmp_script_file = Redata::RED.root.join 'tmp', 'queries', "adj_#{Redata::RED.params[0]}_#{Redata::RED.params[1]}.sql"
+temp_config = OpenStruct.new
+temp_config.query_file = Redata::RED.root.join 'database', 'adjust', "#{Redata::RED.params[1]}.sql"
+temp_config.tmp_file_dir = Redata::RED.root.join "tmp", "adj_#{Redata::RED.params[0]}_#{Redata::RED.params[1]}"
+temp_config.tmp_exec_file = Redata::RED.root.join "tmp", "adj_#{Redata::RED.params[0]}_#{Redata::RED.params[1]}", "exec.sql"
 
-Redata::Log.action "QUERY: Run query file [#{query_file.relative_path_from Redata::RED.root}] in #{Redata::RED.params[0]}"
-Redata::Parser.gen_adjust_file query_file, tmp_script_file
+Dir.mkdir temp_config.tmp_file_dir unless temp_config.tmp_file_dir.exist?
+
+Redata::Log.action "ADJUST<#{Redata::RED.params[0]}>: use [#{temp_config.query_file.relative_path_from Redata::RED.root}]"
+Redata::Parser.gen_adjust_query temp_config
 
 if Redata::RED.params[0] == "redshift"
-	Redata::DATABASE.connect_with_file tmp_script_file
+	Redata::DATABASE.connect_redshift temp_config
 else
-	Redata::DATABASE.connect_mysql_with_file tmp_script_file, Redata::RED.params[0], Redata::RED.params[2]
+	Redata::DATABASE.connect_mysql temp_config.tmp_exec_file, Redata::RED.params[0], Redata::RED.params[2]
 end
 
+FileUtils.rm_r temp_config.tmp_file_dir if temp_config.tmp_file_dir.exist?

data/bin/notice

@@ -6,15 +6,13 @@ require File.expand_path '../../lib/redata/notice', __FILE__
 
 
 
-if ['update'].include? Redata::RED.params[0]
+if ['log', 'mention'].include? Redata::RED.params[0]
 	notice = Redata::Notice.new
 	case Redata::RED.params[0]
-	when 'update'
-		if Redata::RED.production?
-			notice.send "今日の自動更新を完了しました！\n今のデータ期間は `2015-07-29 ~ #{Redata::RED.date_days_ago 3}` になってます"
-		else
-			Redata::Log.warning "WARNING: Could send notice only in production env"
-		end
+	when 'log'
+		notice.log Redata::RED.params[1], Redata::RED.params[2]
+	when 'mention'
+		notice.mention Redata::RED.params[1], Redata::RED.params[2]
 	end
 end
 

data/bin/redata

@@ -5,7 +5,7 @@ require File.expand_path '../../lib/redata', __FILE__
 require Redata::RED.root.join('config', 'relations.rb').to_s
 
 
-if ['create', 'delete', 'checkout', 'inject'].include? Redata::RED.params[0]
+if ['create', 'delete', 'checkout', 'deploy'].include? Redata::RED.params[0]
 	case Redata::RED.params[0]
 	when 'create'
 		Redata::Task.create_datasource Redata::RED.params[1]
@@ -13,8 +13,8 @@ if ['create', 'delete', 'checkout', 'inject'].include? Redata::RED.params[0]
 		Redata::Task.delete_datasource Redata::RED.params[1]
 	when 'checkout'
 		Redata::Task.checkout_datasource Redata::RED.params[1]
-	when 'inject'
-		Redata::Task.inject Redata::RED.params[1], Redata::RED.params[2]
+	when 'deploy'
+		Redata::Task.deploy_datasource Redata::RED.params[1], Redata::RED.params[2]
 	end
 end
 

data/lib/redata/config.rb

@@ -22,7 +22,8 @@ module Redata
 			# config file
 			@config = YAML.load(ERB.new(File.read(@root.join 'config', 'redata.yml')).result(binding))
 			@s3_config = @config['s3']
-			@s3_config['bucket'] = @s3_config['bucket'][@env]
+			@s3_config['bucket'] += "-dev" unless @env == 'production'
+			@s3_config['region'] = 'ap-northeast-1'
 			@s3_config['host'] = "https://s3-#{@s3_config['region']}.amazonaws.com/#{@s3_config['bucket']}"
 			Aws.config.update({
 				region: @s3_config['region'],
@@ -30,6 +31,7 @@ module Redata
 			})
 			@tz_local = Timezone[@config['timezone']]
 			@slack_token = @config['slack_bot']
+			@keep_tmp = @config['keep_tmp']
 		end
 
 		def development?
@@ -40,8 +42,26 @@ module Redata
 			@env == 'production'
 		end
 
-		def default_start_date
-			@config['start_date']
+		def keep_tmp?
+			@keep_tmp
+		end
+
+		def start_time
+			return @locals[:start_time] if @locals[:start_time]
+			if @is_append
+				@tz_local.utc_to_local(Time.now.utc-@config['append_interval']['start_time']*24*3600).strftime('%Y-%m-%d')
+			else
+				@config['create_interval']['start_time']
+			end
+		end
+
+		def end_time
+			return @locals[:end_time] if @locals[:end_time]
+			if @is_append
+				@tz_local.utc_to_local(Time.now.utc-@config['append_interval']['end_time']*24*3600).strftime('%Y-%m-%d')
+			else
+				@tz_local.utc_to_local(Time.now.utc-@config['create_interval']['end_time']*24*3600).strftime('%Y-%m-%d')
+			end
 		end
 
 		def ssh
@@ -60,11 +80,6 @@ module Redata
 			@tz_local.utc_to_local(Time.now.utc).strftime('%Y-%m-%d %H:%M:%S')
 		end
 
-		def default_append_date
-			# 2 days ago bacause there is only data 2 days ago in redshift
-			@tz_local.utc_to_local(Time.now.utc-2*24*3600).strftime('%Y-%m-%d')
-		end
-
 		def date_days_ago(days)
 			@tz_local.utc_to_local(Time.now.utc-days*24*3600).strftime('%Y-%m-%d')
 		end
@@ -76,9 +91,6 @@ module Redata
 			i = 0
 			while i < argv.count
 				case argv[i]
-				when '-dir'
-					i += 1
-					new_argv[:dir] = argv[i]
 				when '-e'
 					i += 1
 					new_argv[:env] = argv[i]
@@ -86,11 +98,11 @@ module Redata
 					new_argv[:force] = true
 				when '-ssh'
 					new_argv[:ssh] = true
-				when '-append_mode'
+				when '-append'
 					new_argv[:append_mode] = true
 				else
-					if argv[i] =~ /-(.+)/
-						key = argv[i].match(/-(.+)/)[1]
+					if argv[i] =~ /\A-(.+)/
+						key = argv[i].match(/\A-(.+)/)[1]
 						i += 1
 						new_argv[:locals][key.to_sym] = argv[i]
 					else

data/lib/redata/database.rb

@@ -9,59 +9,44 @@ module Redata
 			@ssh = Ssh.new
 		end
 
-		def connect_with_file(file)
+		def connect_redshift(config)
 			cmd = make_redshift_cmd
 			if @ssh.run_with_ssh?
-				@ssh.upload_file file
-				@ssh.run_command "export PGPASSWORD='#{ENV['PGPASSWORD']}';#{cmd} -f ~/tmp/#{file.basename}"
+				@ssh.upload_dir config.tmp_file_dir
+				@ssh.run_command "export PGPASSWORD='#{ENV['PGPASSWORD']}';#{cmd} -f ~/tmp/#{config.tmp_file_dir.basename}/exec.sql"
+				@ssh.remove_dir "~/tmp/#{config.tmp_file_dir.basename}"
 			else
-				system "#{cmd} -f #{file}"
+				system "#{cmd} -f #{config.tmp_exec_file}"
 			end
 		end
 
-		def connect_with_query(query)
-			cmd = make_redshift_cmd
-			if @ssh.run_with_ssh?
-				@ssh.run_command "export PGPASSWORD='#{ENV['PGPASSWORD']}';#{cmd} -c '#{query}'"
-			else
-				system "#{cmd} -c '#{query}'"
-			end
-		end
+		def inject_data(config, stage)
+			target_config = @config['deploy'][config.category.to_s]
+			Log.error! "ERROR: Export config of #{config.category} was not found" unless target_config
 
-		def inject_to_mysql(config, platform)
-			if @ssh.run_with_ssh?
-				@ssh.upload_file config.tmp_data_file, config.name
-				data_file = "~/tmp/#{config.name}"
-			else
-				data_file = config.tmp_data_file
-			end
+			target_config = target_config[stage] if stage
+			Log.error! "ERROR: Export config of #{config.category} for stage #{stage} was not found" unless target_config
 
-			is_append = RED.is_append && config.update_type == :append
-			cmd = "mysqlimport #{make_mysql_cmd_config(config.category.to_s, platform)} #{data_file} --local #{is_append ? '' : '--delete'} --fields-terminated-by='\\t' --fields-enclosed-by='\\\"' --lines-terminated-by='\\n'"
-
-			if @ssh.run_with_ssh?
-				@ssh.run_command cmd
+			if target_config['local_dir']
+				cmd = "mv #{config.tmp_data_file} #{target_config['local_dir']}/#{config.source_name}.tsv"
+			elsif target_config['database']
+				import_params = "--local #{RED.is_append ? '' : '--delete'} --fields-terminated-by='\\t' --fields-enclosed-by='\\\"' --lines-terminated-by='\\n'"
+				cmd = "mysqlimport #{make_mysql_cmd_params(target_config)} #{config.tmp_data_file} #{import_params}"
 			else
-				system "#{cmd}"
+				Log.error! "ERROR: Export config of #{config.category} was not found" unless target_config
 			end
-
+			system cmd
 		end
 
-		def connect_mysql_with_file(query_file, category, platform)
-			if @ssh.run_with_ssh?
-				@ssh.upload_file query_file, query_file.basename
-				data_file = "~/tmp/#{query_file.basename}"
-			else
-				data_file = query_file
-			end
+		def connect_mysql(query_file, category, stage)
+			target_config = @config['deploy'][category.to_s]
+			Log.error! "ERROR: Export config of #{config.category} was not found" unless target_config
 
-			cmd = "mysql #{make_mysql_cmd_config(category, platform)} < #{data_file}"
+			target_config = target_config[stage] if stage
+			Log.error! "ERROR: Export config of #{config.category} for stage #{stage} was not found" unless target_config
 
-			if @ssh.run_with_ssh?
-				@ssh.run_command cmd
-			else
-				system cmd
-			end
+			cmd = "mysql #{make_mysql_cmd_params(target_config)} < #{query_file}"
+			system cmd
 		end
 
 		private
@@ -70,18 +55,8 @@ module Redata
 			return "psql -h #{@config['host']} -p #{REDSHIFT_PORT} -U #{@config['username']} -d #{@config['database']}"
 		end
 
-		def make_mysql_cmd_config(category, platform)
-			export_db_config = @config['export'][category]
-			Log.error! "ERROR: Export config of #{category} was not found in config/database.yml" unless export_db_config
-			if platform
-				if export_db_config[platform]
-					export_db_config = export_db_config[platform]
-				else
-					Log.warning "WARNING: Platform #{platform} was not declared in config/database.yml, ignore platform setting"
-				end
-			end
-
-			return "-h#{export_db_config['host']} -u#{export_db_config['username']} #{export_db_config['password'].empty? ? '' : '-p'+export_db_config['password']} #{export_db_config['database']}"
+		def make_mysql_cmd_params(db_config)
+			return "-h#{db_config['host']} -u#{db_config['username']} #{db_config['password'].empty? ? '' : '-p'+db_config['password']} #{db_config['database']}"
 		end
 
 	end

data/lib/redata/notice.rb

@@ -10,12 +10,25 @@ module Redata
 			Log.error! "ERROR: slack channel #{RED.slack['channel']} not exists" unless channel_exist
 		end
 
-		def send(msg)
+		def log(msg, log=nil)
+			log_content = "```\n#{File.read(log).split("\n").map{|line| line.gsub(/\[0;\d{2};\d{2}m/, '').gsub(/\[0m/, '')}.join("\n")}\n```" if log
 			@slack.chat_postMessage({
 				:channel => RED.slack['channel'],
-				:text => "<!here> #{msg}",
+				:text => "#{msg}\n#{log_content}",
 				:as_user => true
 			})
 		end
+
+		def mention(user_name, msg)
+			@slack.users_list['members'].each do |user|
+				if user['name'] == user_name
+					@slack.chat_postMessage({
+						:channel => RED.slack['channel'],
+						:text => "<@#{user['id']}> #{msg}",
+						:as_user => true
+					})
+				end
+			end
+		end
 	end
 end

data/lib/redata/parser.rb

@@ -1,146 +1,177 @@
 module Redata
 	class Parser
-		INCLUDE_REGEX = /#include (.*)-->(.*)/
-		REF_REGEX = /{([^{}]+)}/
-		REF_SPLIT_REGEX = /\s*{[^{}]+}\s*/
+		COMMENT_REGEX = /-{2}.*/
+		LOAD_REGEX = /#load (.*)->(.*)/
+		IF_REGEX = /\[\s*if ([^\s]*) is ([^\]]*)\]/
+		IFNUL_REGEX = /\[\s*if ([^\s]*) is null\s*\]/
+		ENDIF_REGEX = /\[\s*endif\s*\]/
 		START_TIME_REGEX = /\[start_time\]/
+		END_TIME_REGEX = /\[end_time\]/
 		TIME_OFFSET_REGEX = /\[(\d+) days ago\]/
 		CURRENT_TIME_REGEX = /\[current_time\]/
-		LOCALS_REGEX = /\[([^\[\]]+)\]/
+		LOCALS_REGEX = /\[([^\[\]<>\s]+)\]/
+		LOCALS_LIST_REGEX = /\[<([^\[\]<>\s]+)>\]/
 
-		CONV_TABLE_REGEX = /source:(.*)/
-		CONV_COLUMN_REGEX = /columns:\s*/
-		CONV_SWITCHDEF_REGEX = /(.+){(.*)}/
-		CONV_SWITCH_REGEX = /([^,]+)=>([^,]+)/
-		CONV_TIMESTAMP_REGEX = /\[time_stamp\]/
-
-		def self.gen_redshift_query(config, start_time=nil)
-			Log.error! "ERROR: Query file '#{config.query_file.relative_path_from RED.root}' not exists" unless config.query_file.exist?
+		def self.gen_create_query(config)
+			if config.type == :table
+				self.gen_table_query config
+			elsif config.type == :view
+				self.gen_view_query config
+			end
+		end
 
-			File.open config.tmp_script_file, 'w' do |f|
-				if start_time && config.type == :table
-					f.puts "INSERT INTO #{config.source_name} ("
-				else
-					start_time = RED.default_start_date
-					f.puts "CREATE #{config.type} #{config.source_name} AS ("
-				end
-				self.parse_redshift_file config.query_file, f, start_time
-				f.puts ");"
+		def self.gen_delete_query(config)
+			File.open config.tmp_exec_file, 'w' do |f|
+				f.puts "DROP #{config.type} #{config.source_name} #{RED.is_forced ? 'CASCADE' : 'RESTRICT'};"
 			end
 		end
 
-		def self.gen_export_query(config, start_time=nil)
-			Log.error! "ERROR: Convertor config '#{config.conv_file.relative_path_from RED.root}' not exists" unless config.conv_file.exist?
+		def self.gen_checkout_query(config)
+			Log.error! "ERROR: Only could checkout data from view" unless config.type == :view
 
-			File.open config.tmp_script_file, 'w' do |f|
+			File.open config.tmp_exec_file, 'w' do |f|
 				f.puts "UNLOAD ('"
-				f.puts self.parse_convertor_file config.conv_file
-				f.puts "where date >= \\'#{start_time}\\'" if start_time
+				f.puts "SELECT * FROM #{config.source_name}"
 				f.puts "') to 's3://#{RED.s3['bucket']}/#{config.bucket_file}'"
 				f.puts "CREDENTIALS 'aws_access_key_id=#{RED.s3['aws_access_key_id']};aws_secret_access_key=#{RED.s3['aws_secret_access_key']}'"
 				f.puts "ESCAPE ALLOWOVERWRITE PARALLEL OFF DELIMITER AS '\\t';"
 			end
 		end
 
-		def self.gen_adjust_file(query_file, tmp_script_file)
-			Log.error! "ERROR: Query file '#{query_file.relative_path_from RED.root}' not exists" unless query_file.exist?
+		def self.gen_adjust_query(config)
+			self.parse config.query_file, config.tmp_exec_file, ''
+		end
+
+
+		private
+		def self.gen_table_query(config)
+			Log.error! "ERROR: Relation error" unless config.type == :table
 
-			File.open tmp_script_file, 'w' do |f|
-				self.parse_redshift_file query_file, f, RED.default_start_date
+			tmp_file = config.tmp_file_dir.join "#{config.source_name}.resql"
+			temp_tables = self.parse config.query_file, tmp_file
+
+			File.open config.tmp_exec_file, 'w' do |f|
+				# print temp tables
+				temp_tables.each do |name|
+					f.puts "CREATE TEMP TABLE #{name} AS ("
+					f.puts File.read(config.tmp_file_dir.join "#{name}.resql")
+					f.puts ");"
+				end
+
+				# print create or insert query
+				if RED.is_append
+					f.puts "INSERT INTO #{config.source_name} ("
+				elsif
+					f.puts "CREATE #{config.type} #{config.source_name} AS ("
+				end
+				f.puts File.read tmp_file
+				f.puts ");"
 			end
 		end
 
+		def self.gen_view_query(config)
+			Log.error! "ERROR: Relation error" unless config.type == :view
 
-		private
-		def self.parse_redshift_file(in_file, out, start_time)
-			links = {}
-			File.open(in_file).each.with_index do |line, index|
-				if line =~ INCLUDE_REGEX
-					# parse include syntax
-					res = line.scan(INCLUDE_REGEX).first
-					sub = res[0].gsub /[\s|\'|\"]+/, ''
-					link = res[1].gsub /[\s|:]+/, ''
-					Log.error! "QUERY ERROR: #{in_file.relative_path_from RED.root}:#{index+1}: include query is missing file or alias" if sub.empty? || link.empty?
-					
-					sub_file = in_file.parent.join "_#{sub}.sql"
-					sub_file = RED.root.join 'database', 'shared', "_#{sub}.sql" unless sub_file.exist?
-					Log.error! "QUERY ERROR: #{in_file.relative_path_from RED.root}:#{index+1}: included file _#{sub}.sql could not be found in ./ or {root}/database/shared/" unless sub_file.exist?
-
-					Log.error! "QUERY ERROR: #{in_file.relative_path_from RED.root}:#{index+1}: alias #{link} was declared multiple times" if links[link]
-
-					links[link] = sub_file
-				elsif line =~ REF_REGEX
-					# parse {ref} syntax
-					res = line.scan REF_REGEX
-					refs = res.map{|r| r.first.gsub /\s+/, ''}
-					origins = line.split REF_SPLIT_REGEX
-
-					out.puts origins[0].gsub(';', '')
-					refs.each_with_index do |ref, i|
-						Log.error! "QUERY ERROR: #{in_file}:#{index+1}:\nsub query #{ref} not found." unless links[ref]
-						out.puts "("
-						self.parse_redshift_file links[ref], out, start_time
-						out.puts ") as #{ref}"
-						out.puts origins[i+1].gsub(';', '') if origins[i+1]
+			tmp_file = config.tmp_file_dir.join "#{config.source_name}.resql"
+			temp_tables = self.parse config.query_file, tmp_file
+
+			File.open config.tmp_exec_file, 'w' do |f|
+				f.puts "CREATE #{config.type} #{config.source_name} AS ("
+				temp_tables.each_with_index do |name, index|
+					f.puts "#{index == 0 ? 'WITH' : ','} #{name} AS ("
+					f.puts File.read(config.tmp_file_dir.join "#{name}.resql")
+					f.puts ")"
+				end
+
+				# print create query
+				main = File.read tmp_file
+				unless temp_tables.empty?
+					main.gsub! 'WITH', ','
+					main.gsub! 'with', ','
+				end
+				f.puts main
+				f.puts ");"
+			end
+		end
+
+		def self.parse(in_file, out_file, skip_char=';')
+			Log.error! "ERROR: Query file '#{in_file.relative_path_from RED.root}' not exists" unless in_file.exist?
+
+			temp_tables = []
+			parse_enable = true
+			File.open out_file, 'w' do |out|
+				File.open(in_file).each do |line|
+					# remove comments
+					line.gsub!(COMMENT_REGEX, '')
+					# remove skip_char
+					line.gsub!(skip_char, '')
+					# remove empty line
+					next if !line || line.empty? || line =~ /^\s*$/
+
+					# check if else condition
+					if line =~ IFNUL_REGEX
+						res = line.scan(IFNUL_REGEX).first
+						var = res[0]
+						parse_enable = RED.locals[var.to_sym].nil?
+						next
+					elsif line =~ IF_REGEX
+						res = line.scan(IF_REGEX).first
+						var = res[0]
+						val = res[1].gsub /[\s|\'|\"]+/, ''
+						parse_enable = (RED.locals[var.to_sym] == val)
+						next
+					elsif line =~ ENDIF_REGEX
+						parse_enable = true
+						next
+					end
+					next unless parse_enable
+
+					# compile sub file
+					if line =~ LOAD_REGEX
+						# parse load syntax
+						res = line.scan(LOAD_REGEX).first
+						sub = res[0].gsub /[\s|\'|\"]+/, ''
+						name = res[1].gsub /[\s|:]+/, ''
+						Log.error! "QUERY ERROR: syntax error for load query: #{line}" if sub.empty? || name.empty?
+						
+						sub_file = in_file.parent.join "_#{sub}.red.sql"
+						sub_file = RED.root.join 'database', 'shared', "_#{sub}.rea.sql" unless sub_file.exist?
+						sub_temp_tables = self.parse sub_file, out_file.dirname.join("#{name}.resql")
+						sub_temp_tables.each do |n|
+							temp_tables.push n unless temp_tables.include? n
+						end
+						temp_tables.push name unless temp_tables.include? name
+						next  # load query line can not contain other content
 					end
-				elsif line =~ START_TIME_REGEX
+
 					# parse [start_time] syntax
-					out.puts line.gsub(START_TIME_REGEX, "'#{start_time}'").gsub(';', '')
-				elsif line =~ TIME_OFFSET_REGEX
+					line.gsub! START_TIME_REGEX, "'#{RED.start_time}'" 
+					# parse [end_time] syntax
+					line.gsub! END_TIME_REGEX, "'#{RED.end_time}'"
+					# parse [current_time] syntax
+					line.gsub! CURRENT_TIME_REGEX, "'#{RED.current_time}'"
+
 					# parse [3 days ago]
 					res = line.scan(TIME_OFFSET_REGEX).each do |res|
-						line = line.gsub "[#{res[0]} days ago]", "#{RED.date_days_ago(res[0].to_i)}"
+						line.gsub! "[#{res[0]} days ago]", "'#{RED.date_days_ago(res[0].to_i)}'"
 					end
-					out.puts line
-				elsif line =~ CURRENT_TIME_REGEX
-					line = line.gsub "[current_time]", "#{RED.current_time}"
-					out.puts line
-				elsif line =~ LOCALS_REGEX
 					# parse [locals] syntax
 					line.scan(LOCALS_REGEX).each do |res|
 						key = res.first
 						Log.error! "QUERY ERROR: Local params #{key} was missing." unless RED.locals[key.to_sym]
-						line = line.gsub "[#{key}]", "'#{RED.locals[key.to_sym]}'"
+						line.gsub! "[#{key}]", "'#{RED.locals[key.to_sym]}'"
 					end
-					out.puts line.gsub ';', ''
-				else
-					# other, print absolutely
-					out.puts line.gsub ';', ''
-				end
-			end
-		end
-
-		def self.parse_convertor_file(in_file)
-			is_parsing_column = false
-			columns = []
-			source = ""
-			File.open(in_file).each.with_index do |line, index|
-				if line =~ CONV_TABLE_REGEX
-					# parse table declare
-					res = line.scan(CONV_TABLE_REGEX).first
-					source = res[0].gsub /\s+/, ''
-					is_parsing_column = false
-				elsif line =~ CONV_COLUMN_REGEX
-					is_parsing_column = true
-				elsif is_parsing_column
-					line.gsub! /\s+/, ''
-					if line =~ CONV_SWITCHDEF_REGEX
-						res = line.scan(CONV_SWITCHDEF_REGEX).first
-						res[1].gsub!("'", "\\\\'")
-						switches = res[1].scan CONV_SWITCH_REGEX
-						switches.map! do |m|
-							"when #{m[0]} then #{m[1]}"
-						end
-						columns.push "case #{res[0]} #{switches.join ' '} end as #{res[0]}"
-					elsif line =~ CONV_TIMESTAMP_REGEX
-						columns.push "\\'#{(Time.now+9*3600).strftime("%Y-%m-%d %H:%M:%S")}\\'"
-						columns.push "\\'#{(Time.now+9*3600).strftime("%Y-%m-%d %H:%M:%S")}\\'"
-					else
-						columns.push line.gsub("'", "\\\\'").gsub('NULL', "\\\\'NULL\\\\'") unless line.empty?
+					# parse [<local_list>] syntax
+					line.scan(LOCALS_LIST_REGEX).each do |res|
+						key = res.first
+						Log.error! "QUERY ERROR: Local params #{key} was missing." unless RED.locals[key.to_sym]
+						line = line.gsub "[<#{key}>]", "(#{RED.locals[key.to_sym].split(',').map{|e| "'#{e}'"}.join(',')})"
 					end
+
+					out.puts line.gsub skip_char, ''
 				end
 			end
-			"select #{columns.join ','} from #{source}"
+			temp_tables
 		end
 
 	end

data/lib/redata/relation.rb

@@ -1,6 +1,6 @@
 module Redata
 	class Relation
-		attr_accessor :category, :name, :key, :file, :dir, :type, :update_type
+		attr_accessor :category, :name, :key, :file, :dir, :type
 		def initialize(category, name, setting)
 			@category = category
 			@name = name
@@ -13,5 +13,31 @@ module Redata
 			@category == :main ? @key : "#{@category}_#{@key}".to_sym
 		end
 
+		def source_name
+			@category == :main ? @name : "#{@category}_#{@name}"
+		end
+
+		def query_file
+			query_file = RED.root.join 'database', 'sources'
+			query_file = query_file.join @dir if @dir
+			query_file = query_file.join "#{@file}.red.sql"
+			query_file
+		end
+
+		def tmp_file_dir
+			RED.root.join 'tmp', "#{@category}_#{@name}"
+		end
+
+		def tmp_exec_file
+			self.tmp_file_dir.join "exec.sql"
+		end
+
+		def tmp_mkdir
+			Dir.mkdir self.tmp_file_dir unless self.tmp_file_dir.exist?
+		end
+
+		def tmp_rmdir
+			FileUtils.rm_r self.tmp_file_dir if !RED.keep_tmp? && self.tmp_file_dir.exist?
+		end
 	end
 end

data/lib/redata/relation/table.rb

@@ -3,22 +3,6 @@ module Redata
 		def initialize(category, name, setting)
 			super category, name, setting
 			@type = :table
-			@update_type = setting[:update] || :renewal
-		end
-
-		def source_name
-			@category == :main ? @name : "#{@category}_#{@name}"
-		end
-
-		def query_file
-			query_file = RED.root.join 'database', 'sources'
-			query_file = query_file.join @dir if @dir
-			query_file = query_file.join "#{@file}.sql"
-			query_file
-		end
-
-		def tmp_script_file
-			RED.root.join 'tmp', "queries", "red#{@type}_#{@category}_#{@name}.sql"
 		end
 	end
 end

data/lib/redata/relation/view.rb

@@ -3,22 +3,14 @@ module Redata
 		def initialize(category, name, setting)
 			super category, name, setting
 			@type = :view
-			@update_type = :renewal
 		end
 
-		def source_name
-			@category == :main ? @name : "#{@category}_#{@name}"
+		def bucket_file
+			"#{RED.end_time}/#{@category}/#{@name}.tsv"
 		end
 
-		def query_file
-			query_file = RED.root.join 'database', 'sources'
-			query_file = query_file.join @dir if @dir
-			query_file = query_file.join "#{@file}.sql"
-			query_file
-		end
-
-		def tmp_script_file
-			RED.root.join 'tmp', "queries", "red#{@type}_#{@category}_#{@name}.sql"
+		def tmp_data_file
+			self.tmp_file_dir.join "#{@name}.tsv"
 		end
 	end
 end

data/lib/redata/requires.rb

@@ -18,7 +18,6 @@ require 'redata/bucket'
 require 'redata/relation'
 require 'redata/relation/table'
 require 'redata/relation/view'
-require 'redata/relation/export'
 require 'redata/schema'
 require 'redata/parser'
 require 'redata/tasks'

data/lib/redata/ssh.rb

@@ -17,12 +17,16 @@ module Redata
 			return false
 		end
 
-		def upload_file(file, target_file=nil)
-			system "scp -i #{@ssh['IdentityFile']} #{file} #{@ssh['User']}@#{@ssh['HostName']}:~/tmp/#{target_file}"
+		def upload_dir(dir)
+			system "scp -r -i #{@ssh['IdentityFile']} #{dir} #{@ssh['User']}@#{@ssh['HostName']}:~/tmp/"
 		end
 
 		def run_command(cmd)
 			system "ssh -i #{@ssh['IdentityFile']} #{@ssh['User']}@#{@ssh['HostName']} \"#{cmd}\""
 		end
+
+		def remove_dir(dir)
+			system "ssh -i #{@ssh['IdentityFile']} #{@ssh['User']}@#{@ssh['HostName']} \"rm -rf #{dir}\""
+		end
 	end
 end

data/lib/redata/tasks.rb

@@ -8,59 +8,56 @@ module Redata
 
 		def self.create_datasource(key)
 			self.parse_key(key, [:table, :view]).each do |config|
-				if RED.is_append && config.update_type == :append
-					start_time = RED.locals[:start_time] || RED.default_append_date
-					Parser.gen_redshift_query config, start_time
-					Log.action "QUERY: Append data after #{start_time} into [#{config.source_name}]"
-					DATABASE.connect_with_file config.tmp_script_file
+				config.tmp_mkdir
+				Parser.gen_create_query config
+				if RED.is_append
+					dLog.action "APPEND<#{config.type}>: data(#{RED.start_time} ~ #{RED.end_time}) into [#{config.source_name}]"
 				else
-					Parser.gen_redshift_query config
-					Log.action "QUERY: Create #{config.type} [#{config.source_name}]"
-					DATABASE.connect_with_file config.tmp_script_file
+					Log.action "CREATE<#{config.type}>: [#{config.source_name}]"
 				end
+				DATABASE.connect_redshift config
+				config.tmp_rmdir
 			end
 		end
 
 		def self.delete_datasource(key)
 			self.parse_key(key, [:table, :view]).reverse.each do |config|
-				unless RED.is_append && config.update_type == :append
-					Log.action "QUERY: Drop #{config.type} [#{config.source_name}]"
-					Log.warning "WARNING: CASCADE mode will also drop other sources that depend on this #{config.type}" if RED.is_forced
-					DATABASE.connect_with_query "DROP #{config.type} #{config.source_name} #{RED.is_forced ? 'CASCADE' : 'RESTRICT'}"
-				end
+				config.tmp_mkdir
+				Parser.gen_delete_query config
+				Log.action "DROP<#{config.type}>: [#{config.source_name}]"
+				Log.warning "WARNING: CASCADE mode will also drop other views that depend on this" if RED.is_forced
+				DATABASE.connect_redshift config
+				config.tmp_rmdir
 			end
 		end
 
 		def self.checkout_datasource(key)
-			self.parse_key(key, [:export]).each do |config|
-				if RED.is_append && config.update_type == :append
-					start_time = RED.locals[:start_time] || RED.default_append_date
-					Parser.gen_export_query config, start_time
-					Log.action "QUERY: Checkout data after #{start_time} to bucket [#{config.bucket_file}]"
-				else
-					Parser.gen_export_query config
-					Log.action "QUERY: Checkout data to bucket [#{config.bucket_file}]"
-				end
-				DATABASE.connect_with_file config.tmp_script_file
+			self.parse_key(key, [:view]).each do |config|
+				config.tmp_mkdir
+				Parser.gen_checkout_query config
+				Log.action "CHECKOUT<#{config.category}>: to bucket [#{config.bucket_file}]"
+				DATABASE.connect_redshift config
+
 				bucket = S3Bucket.new
 				bucket.move "#{config.bucket_file}000", config.bucket_file
+				config.tmp_rmdir
 			end
 		end
 
-		def self.inject(key, platform=nil)
-			self.parse_key(key, [:export]).each do |config|
-				Log.action "BUCKET: Make [#{config.bucket_file}] public"
+		def self.deploy_datasource(key, stage)
+			self.parse_key(key, [:view]).each do |config|
+				config.tmp_mkdir
 				bucket = S3Bucket.new
 				bucket.make_public config.bucket_file, true
 
-				Log.action "DOWNLOAD: Downlaod [#{config.bucket_file}] from bucket"
+				Log.action "DOWNLOAD<bucket>: from [#{config.bucket_file}]"
 				system "wget #{RED.s3['host']}/#{config.bucket_file} -O #{config.tmp_data_file} --quiet"
 
-				Log.action "BUCKET: Make [#{config.bucket_file}] private"
 				bucket.make_public config.bucket_file, false
 
-				Log.action "QUERY: Inject data to [#{config.name}] of #{config.category}"
-				DATABASE.inject_to_mysql config, platform
+				Log.action "INJECT<#{config.category}>: with [#{config.name}] #{stage ? 'for stage '+stage : ''}"
+				DATABASE.inject_data config, stage
+				config.tmp_rmdir
 			end
 		end
 
@@ -73,7 +70,7 @@ module Redata
 			configs = @@schema.category_configs(key, types)
 			if configs.empty?
 				config = @@schema.config_with key if key
-				Log.error! "ERROR: Data source relation #{key} was not defined in config/relations.rb" unless config
+				Log.error! "ERROR: Data source relation #{key} was not defined" unless config
 				configs.push config
 			end
 			configs

data/lib/redata/version.rb

@@ -1,3 +1,3 @@
 module Redata
-  VERSION = "0.1.0"
+  VERSION = "1.0.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redata
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 1.0.0
 platform: ruby
 authors:
 - goshan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-12-09 00:00:00.000000000 Z
+date: 2017-02-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -136,7 +136,6 @@ files:
 - lib/redata/notice.rb
 - lib/redata/parser.rb
 - lib/redata/relation.rb
-- lib/redata/relation/export.rb
 - lib/redata/relation/table.rb
 - lib/redata/relation/view.rb
 - lib/redata/requires.rb

data/lib/redata/relation/export.rb

@@ -1,30 +0,0 @@
-module Redata
-	class Export < Relation
-		def initialize(category, name, setting)
-			super category, name, setting
-			@type = :export
-			@update_type = setting[:update] || :renewal
-		end
-
-		def conv_file
-			conv_file = RED.root.join 'database', 'convertors'
-			conv_file = conv_file.join @dir if @dir
-			conv_file = conv_file.join "#{@file}.conv"
-			conv_file
-		end
-
-		def tmp_script_file
-			RED.root.join 'tmp', "queries", "red#{@type}_#{@category}_#{@name}.sql"
-		end
-
-		def tmp_data_file
-			RED.root.join 'tmp', "data", "#{@name}.tsv"
-		end
-
-		def bucket_file
-			bucket_dir = RED.default_append_date
-			bucket_dir = RED.locals[:start_time] if RED.is_append && @update_type == :append && RED.locals[:start_time]
-			"#{bucket_dir}/#{@category}/#{@name}.tsv"
-		end
-	end
-end