redata 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 58c04577fb55ac865dda405347811acf421dac3e
+  data.tar.gz: a1a33fb66a93f198aca56731417c2a2d90b65192
+SHA512:
+  metadata.gz: 199b5da361e782c96ba4dbb1c5e465ca3fd7e9a8f4471776efb0915408ec9f43428aadfda25f5037e883c4798bbec4633c2e0aa7ddb76d59daafc3e8c0ed56dc
+  data.tar.gz: 551db6a70bbcdfd29a39aed84d69258d883afcbede934bc5f7fc08ed165b8c3978671f2cba2e8d26ee3bc5a6a8349cc4cde86028824241b01dfce1fa6e6dd092

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in redata.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 goshan_tvi
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,235 @@
+# Redata
+
+Help you to controll data process in redshift with easy query and command line
+
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'redata'
+```
+
+And then execute:
+
+    $ bundle install
+
+Or install it yourself as:
+
+    $ gem install redata
+
+## Usage
+
+### Config
+
++ config `config/redata.yml` for general setting
+
+```YAML
+start_date: "2016-04-04"  # default data start date
+timezone: "Asia/Tokyo"
+s3:
+  aws_access_key_id: {key_id}
+  aws_secret_access_key: {key_secret}
+  region: {s3_region}
+  bucket:
+	production: {bucket_name}
+	development: {bucket_name}
+ssh:  # this setting will be used in ssh mode when you access private database
+  HostName: {gateway_host}
+  IdentityFile: {~/.ssh/key.pem}
+  User: {username}
+slack_bot:  # this setting will be used for slack notice push
+  token: {bot_token}
+  channel: {slack_channel}
+```
+
++ config `config/database.yml` for development and production environment in redshift database  
+eg.
+
+```YAML
+development:
+  host: localhost
+  username: user
+  password: ''
+  database: dev
+  export: # target platform db(mysql) which export data to
+    app:  # platform name
+      username: root
+      password: ''
+      host: localhost
+      database: app
+```
+
++ config `config/relations.rb` for data object in redshift and exporting process to mysql  
+eg.
+
+```RUBY
+Redata::Task.schema.config do
+  # Example of declaring a global table
+  table 'table_name'
+  # This declaration means
+  #   query file: database/sources/table_name.sql
+  #   redshift table: table_name
+  #   update type: renewal, delete and re-create when update
+  #   key used in command line: table_name
+  
+  # Example of declaring a global table with customizing options
+  table 'new_table_name', :dir => 'dir', :file => 'query_file', :update => :append, :as => :alias
+  # This declaration means
+  #   query file: database/sources/dir/query_file.sql
+  #   redshift table: new_table_name
+  #   update type: append, only appending to existing table
+  #   key used in command line: alias
+  
+  # view is same to table but the update type only has renewal mode
+  table 'view_name'
+  table 'new_view_name', :dir => 'dir', :file => 'query_file_oth', :as => :alias_oth
+  
+  # Example of declaring with category
+  category :test_category do
+    table 'test_table'
+    # This declaration means
+    #   query file: database/sources/test_category/test_table.sql
+    #   redshift table: test_category_test_table
+    #   update type: renewal
+    #   key used in command line: test_category_test_table
+    
+    table 'test_table_oth', :dir => 'dir', :file => 'query_file_oth', :update => append, :as => :alias_oth
+    # This declaration means
+    #   query file: database/sources/dir/query_file_oth.sql
+    #   redshift table: test_category_test_table
+    #   update type: append
+    #   key used in command line: test_category_alias_oth
+    
+    # view is same to table without appending update type
+    view 'test_view'
+    view 'test_view_oth', :dir => 'dir', :file => 'query_file_oth', :as => :alias_view_oth
+    
+    #Example of convertor declaration
+    export 'test_export'
+    # This declaration means
+    #   convertor file: database/convertors/test_category/test_export.conv
+    #   target mysql database name: test_category (Also see: export config in config/database.yml{:export})
+    #   target mysql table: test_export
+    #   update type: renewal, delete all records and insert new records
+    #   key used in command line: test_category_test_export
+    
+    #Example of convertor declaration
+    export 'test_export', :dir => 'dir', :file => 'conv_file', :update => 'append', :as => 'alias_export'
+    # This declaration means
+    #   convertor file: database/convertors/dir/conv_file.conv
+    #   target mysql database name: test_category
+    #   target mysql table: test_export
+    #   update type: append, append insert new records without deleting
+    #   key used in command line: test_category_alias_export
+  end
+
+end
+```
+
+### Query file
+
+Query file was used for create table of view in redshift. It is almost like PostgreSQL file but with same new feature. And you have no need to write a create table/view query, the result after running query file will used to create a new table/view, for table, if you use append mode, the result will only be append-inserted to table.  
+eg.
+
+```SQL
+-- query file in data/sources/...
+
+#include 'sub_query_a' --> :a  -- include a sub query as object a from _sub_query_a.sql in same folder or database/shared/
+#include 'sub_query_b' --> :b
+
+
+select a.col1, a.col2, b.col1, b.col2, b.col3
+from {a}  -- use object a included from sub query file '_sub_query_a.sql'
+join {b} on b.col1 = a.col1
+-- If in append mode and this table was setted appending update type, then start_time getting from command input such as `-start_time 2016-11-08` will be used here. When missing input this param, as default [2 days ago] will be used.
+-- Or if not append mode, start_date will be used as default (Also see config/redata.yml).  set start_time when running command , if missing in command, default_start_date will be used 
+where a.col1 >= [start_time]
+-- current time in setted timezone will be used (About timezon, also see config/redata.yml)
+and a.col1 <= [current_time]
+-- some params getting from command input such as `-param_from_command param_value`
+and a.col2 = [param_from_command]
+-- x days before today, x will be a integer
+and b.col3 >= [x days ago]
+```
+
+### Convertor config file
+
+Convertor file was used to generate a select query to get data from redshift and unload to S3. But you have no need to wirte a unload query. If you are using append mode, only data 2 days ago will be select.   
+eg.
+
+```
+source: redshift_source_table_or_view
+columns:
+	cm_id
+	segment_type{'C' => 0, 'T' => 1, 'M1' => 2, 'M2' => 3, 'M3' => 4, 'F1' => 5, 'F2' => 6, 'F3' => 7}
+	v
+	e
+	base_ai
+	sample_num
+	grp
+```
+
+> convertor config file in `data/convertors/...`  
+> `source` means the source table in redshift  
+> `columns` means the source columns in source table  
+
+### Command
+
+There are 3 executable file in bin/
+- redata --> manage redshift table/view and export data to mysql
+- adjust --> run some single sql query file in redshift or mysql
+- notice --> push notice to slack etc.
+
+#### redata
+
+Usage: `redata [-options] [action] [object key] {platform}`
++ action
+  - create   --> create a table/view or append data to table in redshift
+  - delete   --> delete a table/view in redshift
+  - checkout --> export data in table/view of redshift into S3
+  - inject   --> import data into mysql table from S3
++ object key --> object will be create/delete/checkout/inject declared in `config/relation.rb`
++ platform   --> when injecting data into mysql, there may be several platform declared in `config/database.yml{:export}` for same database, here is setting which platform to use. *If the platform here could not be found in `database.yml` or have not set platform, the default export will be used.*
++ options
+  - -dir --> project directory, both absolute path and realtive path will be okay. default is current directory.
+  - -e   --> environment: `production`, `development`, etc.
+  - -f   --> force mode, use `CADCASE` when removing view or table in redshift
+  - -ssh --> use ssh accessing to private database with ssh config in `config/redata.yml`
+  - -append_mode  --> use `append_mode`, the objects in relations.rb with appending update type will go to appending operation. 
+    + delete will only delete objects with renewal update type
+    + create will append-insert data after `-start_time`(set in command) or default `2 days ago` for appending update type, still create table/view for renewal type
+    + checkout will only fetch data after `-start_time` or default `2 days ago` to upload to S3, renewal type will still be uploaded all data
+    + inject will insert data to mysql without `--delete` option, renewal still delete all firstly
+  - other options  --> some params will be used in query file when declared, such `start_time`
+
+#### adjust
+
+Use adjust when you just want to run a query file without declaring in `config/relations.rb`
+Usage: `adjust [-options] [database] [query file] {platform}`
++ database   --> `redshift` or database declared in `config/database.yml{export}`
++ query file --> query file which will be run in `database/adjust/`, **without extends `.sql`**
++ platform   --> same to `redata`
++ options
+  - -dir --> project directory, both absolute path and realtive path will be okay. default is current directory.
+  - -e   --> environment: `production`, `development`, etc.
+  - -ssh --> use ssh accessing to private database with ssh config in `config/redata.yml`
+  - other options  --> some params will be used in query file when declared, such `start_time`
+
+#### notice
+
+Usage: `notice [-options] [action]`
++ action: currently, there is only `update` action which means send 'finish updating' message to slack
++ options
+  - -e   --> environment: `production`, `development`, etc. **Only production could send notice**
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/goshan/redata.
+
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
+

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+task :default => :spec

data/bin/adjust

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+
+require File.expand_path '../../lib/redata', __FILE__
+
+
+query_file = Redata::RED.root.join 'database', 'adjust', "#{Redata::RED.params[1]}.sql"
+tmp_script_file = Redata::RED.root.join 'tmp', 'queries', "adj_#{Redata::RED.params[0]}_#{Redata::RED.params[1]}.sql"
+
+Redata::Log.action "QUERY: Run query file [#{query_file.relative_path_from Redata::RED.root}] in #{Redata::RED.params[0]}"
+Redata::Parser.gen_adjust_file query_file, tmp_script_file
+
+if Redata::RED.params[0] == "redshift"
+	Redata::DATABASE.connect_with_file tmp_script_file
+else
+	Redata::DATABASE.connect_mysql_with_file tmp_script_file, Redata::RED.params[0], Redata::RED.params[2]
+end
+

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "redata"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/notice

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+
+require 'slack'
+require File.expand_path '../../lib/redata', __FILE__
+require File.expand_path '../../lib/redata/notice', __FILE__
+
+
+
+if ['update'].include? Redata::RED.params[0]
+	notice = Redata::Notice.new
+	case Redata::RED.params[0]
+	when 'update'
+		if Redata::RED.production?
+			notice.send "今日の自動更新を完了しました！\n今のデータ期間は `2015-07-29 ~ #{Redata::RED.date_days_ago 3}` になってます"
+		else
+			Redata::Log.warning "WARNING: Could send notice only in production env"
+		end
+	end
+end
+

data/bin/redata

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+
+require File.expand_path '../../lib/redata', __FILE__
+
+require Redata::RED.root.join('config', 'relations.rb').to_s
+
+
+if ['create', 'delete', 'checkout', 'inject'].include? Redata::RED.params[0]
+	case Redata::RED.params[0]
+	when 'create'
+		Redata::Task.create_datasource Redata::RED.params[1]
+	when 'delete'
+		Redata::Task.delete_datasource Redata::RED.params[1]
+	when 'checkout'
+		Redata::Task.checkout_datasource Redata::RED.params[1]
+	when 'inject'
+		Redata::Task.inject Redata::RED.params[1], Redata::RED.params[2]
+	end
+end
+

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/lib/redata.rb

@@ -0,0 +1,5 @@
+require "redata/version"
+require "redata/requires"
+
+module Redata
+end

data/lib/redata/bucket.rb

@@ -0,0 +1,27 @@
+module Redata
+	class S3Bucket
+		def initialize
+			s3 = Aws::S3::Resource.new
+			@bucket = s3.bucket RED.s3['bucket']
+		end
+
+		def move(source, target)
+			from = @bucket.object source
+			to = @bucket.object target
+			from.move_to to if from.exists?
+		end
+
+		def exist?(file)
+			@bucket.object(file).exists?
+		end
+
+		def delete(file)
+			@bucket.object(file).delete if exist?(file)
+		end
+
+		def make_public(file, is_public)
+			acl = is_public ? 'public-read' : 'private'
+			@bucket.object(file).acl.put({:acl => acl}) if exist?(file)
+		end
+	end
+end

data/lib/redata/config.rb

@@ -0,0 +1,108 @@
+module Redata
+	class Config
+		attr_accessor :root, :env, :is_forced, :is_ssh, :is_append, :params, :locals
+
+		def initialize(argv=nil)
+			# params
+			@argv = parse_params argv
+			if !@argv[:dir]
+				@root = Pathname.new(Dir.pwd)
+			elsif Pathname.new(@argv[:dir]).absolute?
+				@root = Pathname.new(@argv[:dir])
+			else
+				@root = Pathname.new(Dir.pwd).join @argv[:dir]
+			end
+			@env = @argv[:env] || 'development'
+			@is_forced = @argv[:force]
+			@is_ssh = @argv[:ssh]
+			@is_append = @argv[:append_mode]
+			@locals = @argv[:locals]
+			@params = @argv[:params]
+
+			# config file
+			@config = YAML.load(ERB.new(File.read(@root.join 'config', 'redata.yml')).result(binding))
+			@s3_config = @config['s3']
+			@s3_config['bucket'] = @s3_config['bucket'][@env]
+			@s3_config['host'] = "https://s3-#{@s3_config['region']}.amazonaws.com/#{@s3_config['bucket']}"
+			Aws.config.update({
+				region: @s3_config['region'],
+				credentials: Aws::Credentials.new(@s3_config['aws_access_key_id'], @s3_config['aws_secret_access_key'])
+			})
+			@tz_local = Timezone[@config['timezone']]
+			@slack_token = @config['slack_bot']
+		end
+
+		def development?
+			@env == 'development'
+		end
+
+		def production?
+			@env == 'production'
+		end
+
+		def default_start_date
+			@config['start_date']
+		end
+
+		def ssh
+			@config['ssh']
+		end
+
+		def s3
+			@s3_config
+		end
+
+		def slack
+			@slack_token
+		end
+
+		def current_time
+			@tz_local.utc_to_local(Time.now.utc).strftime('%Y-%m-%d %H:%M:%S')
+		end
+
+		def default_append_date
+			# 2 days ago bacause there is only data 2 days ago in redshift
+			@tz_local.utc_to_local(Time.now.utc-2*24*3600).strftime('%Y-%m-%d')
+		end
+
+		def date_days_ago(days)
+			@tz_local.utc_to_local(Time.now.utc-days*24*3600).strftime('%Y-%m-%d')
+		end
+
+		private
+		def parse_params(argv)
+			new_argv = {:params => [], :locals => {}}
+			return new_argv unless argv
+			i = 0
+			while i < argv.count
+				case argv[i]
+				when '-dir'
+					i += 1
+					new_argv[:dir] = argv[i]
+				when '-e'
+					i += 1
+					new_argv[:env] = argv[i]
+				when '-f'
+					new_argv[:force] = true
+				when '-ssh'
+					new_argv[:ssh] = true
+				when '-append_mode'
+					new_argv[:append_mode] = true
+				else
+					if argv[i] =~ /-(.+)/
+						key = argv[i].match(/-(.+)/)[1]
+						i += 1
+						new_argv[:locals][key.to_sym] = argv[i]
+					else
+						new_argv[:params].push argv[i]
+					end
+				end
+				i += 1
+			end
+			new_argv
+		end
+	end
+
+	RED = Config.new ARGV
+end
+

data/lib/redata/database.rb

@@ -0,0 +1,92 @@
+module Redata
+	class DataBase
+		REDSHIFT_PORT = 5439
+
+		def initialize
+			@config = YAML.load(ERB.new(File.read(RED.root.join 'config', 'database.yml')).result(binding))[RED.env]
+			Log.error! "ERROR: Database of #{RED.env} was not declared in config/database.yml" unless @config
+
+			@ssh = Ssh.new
+		end
+
+		def connect_with_file(file)
+			cmd = make_redshift_cmd
+			if @ssh.run_with_ssh?
+				@ssh.upload_file file
+				@ssh.run_command "export PGPASSWORD='#{ENV['PGPASSWORD']}';#{cmd} -f ~/tmp/#{file.basename}"
+			else
+				system "#{cmd} -f #{file}"
+			end
+		end
+
+		def connect_with_query(query)
+			cmd = make_redshift_cmd
+			if @ssh.run_with_ssh?
+				@ssh.run_command "export PGPASSWORD='#{ENV['PGPASSWORD']}';#{cmd} -c '#{query}'"
+			else
+				system "#{cmd} -c '#{query}'"
+			end
+		end
+
+		def inject_to_mysql(config, platform)
+			if @ssh.run_with_ssh?
+				@ssh.upload_file config.tmp_data_file, config.name
+				data_file = "~/tmp/#{config.name}"
+			else
+				data_file = config.tmp_data_file
+			end
+
+			is_append = RED.is_append && config.update_type == :append
+			cmd = "mysqlimport #{make_mysql_cmd_config(config.category.to_s, platform)} #{data_file} --local #{is_append ? '' : '--delete'} --fields-terminated-by='\\t' --fields-enclosed-by='\\\"' --lines-terminated-by='\\n'"
+
+			if @ssh.run_with_ssh?
+				@ssh.run_command cmd
+			else
+				system "#{cmd}"
+			end
+
+		end
+
+		def connect_mysql_with_file(query_file, category, platform)
+			if @ssh.run_with_ssh?
+				@ssh.upload_file query_file, query_file.basename
+				data_file = "~/tmp/#{query_file.basename}"
+			else
+				data_file = query_file
+			end
+
+			cmd = "mysql #{make_mysql_cmd_config(category, platform)} < #{data_file}"
+
+			if @ssh.run_with_ssh?
+				@ssh.run_command cmd
+			else
+				system cmd
+			end
+		end
+
+		private
+		def make_redshift_cmd
+			ENV['PGPASSWORD'] = @config['password']
+			return "psql -h #{@config['host']} -p #{REDSHIFT_PORT} -U #{@config['username']} -d #{@config['database']}"
+		end
+
+		def make_mysql_cmd_config(category, platform)
+			export_db_config = @config['export'][category]
+			Log.error! "ERROR: Export config of #{category} was not found in config/database.yml" unless export_db_config
+			if platform
+				if export_db_config[platform]
+					export_db_config = export_db_config[platform]
+				else
+					Log.warning "WARNING: Platform #{platform} was not declared in config/database.yml, ignore platform setting"
+				end
+			end
+
+			return "-h#{export_db_config['host']} -u#{export_db_config['username']} #{export_db_config['password'].empty? ? '' : '-p'+export_db_config['password']} #{export_db_config['database']}"
+		end
+
+	end
+
+	DATABASE = DataBase.new
+end
+
+

data/lib/redata/log.rb

@@ -0,0 +1,20 @@
+module Redata
+	class Log
+		def self.error!(msg)
+			puts "[#{RED.current_time}] #{msg.red}"
+			exit 1
+		end
+
+		def self.warning(msg)
+			puts "[#{RED.current_time}] #{msg.yellow}"
+		end
+
+		def self.action(msg)
+			puts "[#{RED.current_time}] #{msg.cyan}"
+		end
+
+		def self.log(msg)
+			puts "[#{RED.current_time}] #{msg}"
+		end
+	end
+end

data/lib/redata/notice.rb

@@ -0,0 +1,21 @@
+module Redata
+	class Notice
+		def initialize
+			Log.error! "ERROR: slack bot token missing" if !RED.slack['token'] || RED.slack['token'].empty?
+			@slack = Slack::Client.new :token => RED.slack['token']
+			channel_exist = false
+			@slack.channels_list['channels'].each do |channel|
+				channel_exist = true if channel['name'] == RED.slack['channel']
+			end
+			Log.error! "ERROR: slack channel #{RED.slack['channel']} not exists" unless channel_exist
+		end
+
+		def send(msg)
+			@slack.chat_postMessage({
+				:channel => RED.slack['channel'],
+				:text => "<!here> #{msg}",
+				:as_user => true
+			})
+		end
+	end
+end

data/lib/redata/parser.rb

@@ -0,0 +1,147 @@
+module Redata
+	class Parser
+		INCLUDE_REGEX = /#include (.*)-->(.*)/
+		REF_REGEX = /{([^{}]+)}/
+		REF_SPLIT_REGEX = /\s*{[^{}]+}\s*/
+		START_TIME_REGEX = /\[start_time\]/
+		TIME_OFFSET_REGEX = /\[(\d+) days ago\]/
+		CURRENT_TIME_REGEX = /\[current_time\]/
+		LOCALS_REGEX = /\[([^\[\]]+)\]/
+
+		CONV_TABLE_REGEX = /source:(.*)/
+		CONV_COLUMN_REGEX = /columns:\s*/
+		CONV_SWITCHDEF_REGEX = /(.+){(.*)}/
+		CONV_SWITCH_REGEX = /([^,]+)=>([^,]+)/
+		CONV_TIMESTAMP_REGEX = /\[time_stamp\]/
+
+		def self.gen_redshift_query(config, start_time=nil)
+			Log.error! "ERROR: Query file '#{config.query_file.relative_path_from RED.root}' not exists" unless config.query_file.exist?
+
+			File.open config.tmp_script_file, 'w' do |f|
+				if start_time && config.type == :table
+					f.puts "INSERT INTO #{config.source_name} ("
+				else
+					start_time = RED.default_start_date
+					f.puts "CREATE #{config.type} #{config.source_name} AS ("
+				end
+				self.parse_redshift_file config.query_file, f, start_time
+				f.puts ");"
+			end
+		end
+
+		def self.gen_export_query(config, start_time=nil)
+			Log.error! "ERROR: Convertor config '#{config.conv_file.relative_path_from RED.root}' not exists" unless config.conv_file.exist?
+
+			File.open config.tmp_script_file, 'w' do |f|
+				f.puts "UNLOAD ('"
+				f.puts self.parse_convertor_file config.conv_file
+				f.puts "where date >= \\'#{start_time}\\'" if start_time
+				f.puts "') to 's3://#{RED.s3['bucket']}/#{config.bucket_file}'"
+				f.puts "CREDENTIALS 'aws_access_key_id=#{RED.s3['aws_access_key_id']};aws_secret_access_key=#{RED.s3['aws_secret_access_key']}'"
+				f.puts "ESCAPE ALLOWOVERWRITE PARALLEL OFF DELIMITER AS '\\t';"
+			end
+		end
+
+		def self.gen_adjust_file(query_file, tmp_script_file)
+			Log.error! "ERROR: Query file '#{query_file.relative_path_from RED.root}' not exists" unless query_file.exist?
+
+			File.open tmp_script_file, 'w' do |f|
+				self.parse_redshift_file query_file, f, RED.default_start_date
+			end
+		end
+
+
+		private
+		def self.parse_redshift_file(in_file, out, start_time)
+			links = {}
+			File.open(in_file).each.with_index do |line, index|
+				if line =~ INCLUDE_REGEX
+					# parse include syntax
+					res = line.scan(INCLUDE_REGEX).first
+					sub = res[0].gsub /[\s|\'|\"]+/, ''
+					link = res[1].gsub /[\s|:]+/, ''
+					Log.error! "QUERY ERROR: #{in_file.relative_path_from RED.root}:#{index+1}: include query is missing file or alias" if sub.empty? || link.empty?
+					
+					sub_file = in_file.parent.join "_#{sub}.sql"
+					sub_file = RED.root.join 'database', 'shared', "_#{sub}.sql" unless sub_file.exist?
+					Log.error! "QUERY ERROR: #{in_file.relative_path_from RED.root}:#{index+1}: included file _#{sub}.sql could not be found in ./ or {root}/database/shared/" unless sub_file.exist?
+
+					Log.error! "QUERY ERROR: #{in_file.relative_path_from RED.root}:#{index+1}: alias #{link} was declared multiple times" if links[link]
+
+					links[link] = sub_file
+				elsif line =~ REF_REGEX
+					# parse {ref} syntax
+					res = line.scan REF_REGEX
+					refs = res.map{|r| r.first.gsub /\s+/, ''}
+					origins = line.split REF_SPLIT_REGEX
+
+					out.puts origins[0].gsub(';', '')
+					refs.each_with_index do |ref, i|
+						Log.error! "QUERY ERROR: #{in_file}:#{index+1}:\nsub query #{ref} not found." unless links[ref]
+						out.puts "("
+						self.parse_redshift_file links[ref], out, start_time
+						out.puts ") as #{ref}"
+						out.puts origins[i+1].gsub(';', '') if origins[i+1]
+					end
+				elsif line =~ START_TIME_REGEX
+					# parse [start_time] syntax
+					out.puts line.gsub(START_TIME_REGEX, "'#{start_time}'").gsub(';', '')
+				elsif line =~ TIME_OFFSET_REGEX
+					# parse [3 days ago]
+					res = line.scan(TIME_OFFSET_REGEX).each do |res|
+						line = line.gsub "[#{res[0]} days ago]", "#{RED.date_days_ago(res[0].to_i)}"
+					end
+					out.puts line
+				elsif line =~ CURRENT_TIME_REGEX
+					line = line.gsub "[current_time]", "#{RED.current_time}"
+					out.puts line
+				elsif line =~ LOCALS_REGEX
+					# parse [locals] syntax
+					line.scan(LOCALS_REGEX).each do |res|
+						key = res.first
+						Log.error! "QUERY ERROR: Local params #{key} was missing." unless RED.locals[key.to_sym]
+						line = line.gsub "[#{key}]", "'#{RED.locals[key.to_sym]}'"
+					end
+					out.puts line.gsub ';', ''
+				else
+					# other, print absolutely
+					out.puts line.gsub ';', ''
+				end
+			end
+		end
+
+		def self.parse_convertor_file(in_file)
+			is_parsing_column = false
+			columns = []
+			source = ""
+			File.open(in_file).each.with_index do |line, index|
+				if line =~ CONV_TABLE_REGEX
+					# parse table declare
+					res = line.scan(CONV_TABLE_REGEX).first
+					source = res[0].gsub /\s+/, ''
+					is_parsing_column = false
+				elsif line =~ CONV_COLUMN_REGEX
+					is_parsing_column = true
+				elsif is_parsing_column
+					line.gsub! /\s+/, ''
+					if line =~ CONV_SWITCHDEF_REGEX
+						res = line.scan(CONV_SWITCHDEF_REGEX).first
+						res[1].gsub!("'", "\\\\'")
+						switches = res[1].scan CONV_SWITCH_REGEX
+						switches.map! do |m|
+							"when #{m[0]} then #{m[1]}"
+						end
+						columns.push "case #{res[0]} #{switches.join ' '} end as #{res[0]}"
+					elsif line =~ CONV_TIMESTAMP_REGEX
+						columns.push "\\'#{(Time.now+9*3600).strftime("%Y-%m-%d %H:%M:%S")}\\'"
+						columns.push "\\'#{(Time.now+9*3600).strftime("%Y-%m-%d %H:%M:%S")}\\'"
+					else
+						columns.push line.gsub("'", "\\\\'").gsub('NULL', "\\\\'NULL\\\\'") unless line.empty?
+					end
+				end
+			end
+			"select #{columns.join ','} from #{source}"
+		end
+
+	end
+end

data/lib/redata/relation.rb

@@ -0,0 +1,17 @@
+module Redata
+	class Relation
+		attr_accessor :category, :name, :key, :file, :dir, :type, :update_type
+		def initialize(category, name, setting)
+			@category = category
+			@name = name
+			@key = setting[:as] || @name.to_sym
+			@file = setting[:file] || @name
+			@dir = setting[:dir] || (@category == :main ? nil : @category.to_s)
+		end
+
+		def global_key
+			@category == :main ? @key : "#{@category}_#{@key}".to_sym
+		end
+
+	end
+end

data/lib/redata/relation/export.rb

@@ -0,0 +1,30 @@
+module Redata
+	class Export < Relation
+		def initialize(category, name, setting)
+			super category, name, setting
+			@type = :export
+			@update_type = setting[:update] || :renewal
+		end
+
+		def conv_file
+			conv_file = RED.root.join 'database', 'convertors'
+			conv_file = conv_file.join @dir if @dir
+			conv_file = conv_file.join "#{@file}.conv"
+			conv_file
+		end
+
+		def tmp_script_file
+			RED.root.join 'tmp', "queries", "red#{@type}_#{@category}_#{@name}.sql"
+		end
+
+		def tmp_data_file
+			RED.root.join 'tmp', "data", "#{@name}.tsv"
+		end
+
+		def bucket_file
+			bucket_dir = RED.default_append_date
+			bucket_dir = RED.locals[:start_time] if RED.is_append && @update_type == :append && RED.locals[:start_time]
+			"#{bucket_dir}/#{@category}/#{@name}.tsv"
+		end
+	end
+end

data/lib/redata/relation/table.rb

@@ -0,0 +1,24 @@
+module Redata
+	class Table < Relation
+		def initialize(category, name, setting)
+			super category, name, setting
+			@type = :table
+			@update_type = setting[:update] || :renewal
+		end
+
+		def source_name
+			@category == :main ? @name : "#{@category}_#{@name}"
+		end
+
+		def query_file
+			query_file = RED.root.join 'database', 'sources'
+			query_file = query_file.join @dir if @dir
+			query_file = query_file.join "#{@file}.sql"
+			query_file
+		end
+
+		def tmp_script_file
+			RED.root.join 'tmp', "queries", "red#{@type}_#{@category}_#{@name}.sql"
+		end
+	end
+end

data/lib/redata/relation/view.rb

@@ -0,0 +1,24 @@
+module Redata
+	class View < Relation
+		def initialize(category, name, setting)
+			super category, name, setting
+			@type = :view
+			@update_type = :renewal
+		end
+
+		def source_name
+			@category == :main ? @name : "#{@category}_#{@name}"
+		end
+
+		def query_file
+			query_file = RED.root.join 'database', 'sources'
+			query_file = query_file.join @dir if @dir
+			query_file = query_file.join "#{@file}.sql"
+			query_file
+		end
+
+		def tmp_script_file
+			RED.root.join 'tmp', "queries", "red#{@type}_#{@category}_#{@name}.sql"
+		end
+	end
+end

data/lib/redata/requires.rb

@@ -0,0 +1,24 @@
+# ruby default lib
+require 'pathname'
+require 'yaml'
+require 'erb'
+
+# gem lib
+require 'json'
+require 'colorize'
+require 'aws-sdk'
+require 'timezone'
+
+# local lib
+require 'redata/config'
+require 'redata/log'
+require 'redata/ssh'
+require 'redata/database'
+require 'redata/bucket'
+require 'redata/relation'
+require 'redata/relation/table'
+require 'redata/relation/view'
+require 'redata/relation/export'
+require 'redata/schema'
+require 'redata/parser'
+require 'redata/tasks'

data/lib/redata/schema.rb

@@ -0,0 +1,64 @@
+module Redata
+	class Schema
+		def initialize
+			@category = :main
+			@order = []
+			@index = {}
+		end
+
+		def config(&block)
+			self.instance_eval &block
+		end
+
+		def category(prefix, &block)
+			@category = prefix
+			self.instance_eval &block
+			@category = :main
+		end
+
+		def view(view, setting={})
+			register view, :view, setting
+		end
+
+		def table(table, setting={})
+			register table, :table, setting
+		end
+
+		def export(target, setting={})
+			register target, :export, setting
+		end
+
+		def insert(target, setting={})
+			register target, :insert, setting
+		end
+
+		def config_with(key)
+			config = @index[key]
+			return nil unless config
+			config
+		end
+
+		def category_configs(category, types=[])
+			res = []
+			@order.each do |global_key|
+				config = @index[global_key]
+				res.push config if (!category || config.category == category) && (types.empty? || types.include?(config.type))
+			end
+			res
+		end
+
+		private
+		def register(name, type, setting={})
+			cla = Redata.const_get type.capitalize
+			relation = cla.new @category, name, setting
+
+			if @index[relation.global_key]
+				Log.log "in #{caller.first}"
+				Log.error! "ERROR: Duplicated view alias '#{global_key}'" 
+			end
+
+			@index[relation.global_key] = relation
+			@order.push relation.global_key
+		end
+	end
+end

data/lib/redata/ssh.rb

@@ -0,0 +1,28 @@
+module Redata
+	class Ssh
+		def initialize
+			@ssh = RED.ssh
+		end
+
+		def run_with_ssh?
+			if RED.is_ssh && @ssh && !RED.production?
+				Log.warning "WARNING: Using gateway server #{@ssh['HostName']}"
+				return true
+			end
+
+			if RED.is_ssh
+				Log.warning "WARNING: SSH config file was not found. Ignore this config." unless @ssh
+				Log.warning "WARNING: Could not use ssh mode in production. Ignore this config." if RED.production?
+			end
+			return false
+		end
+
+		def upload_file(file, target_file=nil)
+			system "scp -i #{@ssh['IdentityFile']} #{file} #{@ssh['User']}@#{@ssh['HostName']}:~/tmp/#{target_file}"
+		end
+
+		def run_command(cmd)
+			system "ssh -i #{@ssh['IdentityFile']} #{@ssh['User']}@#{@ssh['HostName']} \"#{cmd}\""
+		end
+	end
+end

data/lib/redata/tasks.rb

@@ -0,0 +1,84 @@
+module Redata
+	class Task
+		@@schema = Schema.new
+
+		def self.schema
+			@@schema
+		end
+
+		def self.create_datasource(key)
+			self.parse_key(key, [:table, :view]).each do |config|
+				if RED.is_append && config.update_type == :append
+					start_time = RED.locals[:start_time] || RED.default_append_date
+					Parser.gen_redshift_query config, start_time
+					Log.action "QUERY: Append data after #{start_time} into [#{config.source_name}]"
+					DATABASE.connect_with_file config.tmp_script_file
+				else
+					Parser.gen_redshift_query config
+					Log.action "QUERY: Create #{config.type} [#{config.source_name}]"
+					DATABASE.connect_with_file config.tmp_script_file
+				end
+			end
+		end
+
+		def self.delete_datasource(key)
+			self.parse_key(key, [:table, :view]).reverse.each do |config|
+				unless RED.is_append && config.update_type == :append
+					Log.action "QUERY: Drop #{config.type} [#{config.source_name}]"
+					Log.warning "WARNING: CASCADE mode will also drop other sources that depend on this #{config.type}" if RED.is_forced
+					DATABASE.connect_with_query "DROP #{config.type} #{config.source_name} #{RED.is_forced ? 'CASCADE' : 'RESTRICT'}"
+				end
+			end
+		end
+
+		def self.checkout_datasource(key)
+			self.parse_key(key, [:export]).each do |config|
+				if RED.is_append && config.update_type == :append
+					start_time = RED.locals[:start_time] || RED.default_append_date
+					Parser.gen_export_query config, start_time
+					Log.action "QUERY: Checkout data after #{start_time} to bucket [#{config.bucket_file}]"
+				else
+					Parser.gen_export_query config
+					Log.action "QUERY: Checkout data to bucket [#{config.bucket_file}]"
+				end
+				DATABASE.connect_with_file config.tmp_script_file
+				bucket = S3Bucket.new
+				bucket.move "#{config.bucket_file}000", config.bucket_file
+			end
+		end
+
+		def self.inject(key, platform=nil)
+			self.parse_key(key, [:export]).each do |config|
+				Log.action "BUCKET: Make [#{config.bucket_file}] public"
+				bucket = S3Bucket.new
+				bucket.make_public config.bucket_file, true
+
+				Log.action "DOWNLOAD: Downlaod [#{config.bucket_file}] from bucket"
+				system "wget #{RED.s3['host']}/#{config.bucket_file} -O #{config.tmp_data_file} --quiet"
+
+				Log.action "BUCKET: Make [#{config.bucket_file}] private"
+				bucket.make_public config.bucket_file, false
+
+				Log.action "QUERY: Inject data to [#{config.name}] of #{config.category}"
+				DATABASE.inject_to_mysql config, platform
+			end
+		end
+
+
+		private
+		def self.parse_key(key, types)
+			key = key.to_sym if key
+			configs = []
+
+			configs = @@schema.category_configs(key, types)
+			if configs.empty?
+				config = @@schema.config_with key if key
+				Log.error! "ERROR: Data source relation #{key} was not defined in config/relations.rb" unless config
+				configs.push config
+			end
+			configs
+		end
+
+	end
+end
+

data/lib/redata/version.rb

@@ -0,0 +1,3 @@
+module Redata
+  VERSION = "0.1.0"
+end

data/redata.gemspec

@@ -0,0 +1,29 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'redata/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "redata"
+  spec.version       = Redata::VERSION
+  spec.authors       = ["goshan"]
+  spec.email         = ["goshan.hanqiu@gmail.com"]
+
+  spec.summary       = %q{a AWS Redshift data process controller}
+  spec.description   = %q{Controll data process by sub query and easy command line}
+  spec.homepage      = "https://github.com/goshan/redata"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.executables   = ["redata", "adjust", "notice"]
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.12"
+  spec.add_development_dependency "rake", "~> 10.0"
+
+  spec.add_runtime_dependency "json", "~> 2.0"
+  spec.add_runtime_dependency "colorize", "~> 0.8"
+  spec.add_runtime_dependency "aws-sdk", "~> 2.6"
+  spec.add_runtime_dependency "timezone", "~> 1.2"
+  spec.add_runtime_dependency "slack-api", "~> 1.2"
+end

metadata

@@ -0,0 +1,172 @@
+--- !ruby/object:Gem::Specification
+name: redata
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- goshan
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-12-09 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.12'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: colorize
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.6'
+- !ruby/object:Gem::Dependency
+  name: timezone
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+- !ruby/object:Gem::Dependency
+  name: slack-api
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+description: Controll data process by sub query and easy command line
+email:
+- goshan.hanqiu@gmail.com
+executables:
+- redata
+- adjust
+- notice
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/adjust
+- bin/console
+- bin/notice
+- bin/redata
+- bin/setup
+- lib/redata.rb
+- lib/redata/bucket.rb
+- lib/redata/config.rb
+- lib/redata/database.rb
+- lib/redata/log.rb
+- lib/redata/notice.rb
+- lib/redata/parser.rb
+- lib/redata/relation.rb
+- lib/redata/relation/export.rb
+- lib/redata/relation/table.rb
+- lib/redata/relation/view.rb
+- lib/redata/requires.rb
+- lib/redata/schema.rb
+- lib/redata/ssh.rb
+- lib/redata/tasks.rb
+- lib/redata/version.rb
+- redata.gemspec
+homepage: https://github.com/goshan/redata
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: a AWS Redshift data process controller
+test_files: []