redata 1.0.0 → 1.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ba347eb745baeb8cf5ffee03842482bfcc2b98d7
-  data.tar.gz: cb86f147ae139d3e87af8afa6e74a823abe02c02
+  metadata.gz: ef52bd600d45561f6a13e060d1417a00545279ef
+  data.tar.gz: c3c9959096211dae0b9f5ea8379c144f18dca51e
 SHA512:
-  metadata.gz: 47cabaf3d9fcab6058a587c823eeee1a6bc39a33051b7675e176dcd7b812e33678ecd89e2f4b4463747813ade4f287947119623b7ec19b45cbf9dbf316a87dc4
-  data.tar.gz: 13991f7d3daa93f15f31bd1e6654147098c98f36f7d1eb19f2003f58792ede7c9b806be33571593d138db5f2425ef9e63444103b8ff9063d98778e14bfdd9dc7
+  metadata.gz: 463e9ed7bbc5a45d85c9fe1bc45c9811cd7f2ee3d6ab4d0750a274552c48efc50942eeb3a32607e9d388968a0d70301dfeb5165747f9d63141a9e352a98bd382
+  data.tar.gz: 6bb1724724771231af491b76cbc13fd52ef29aabe71bdb5d6812f25f04700bf5317f6bda31ffb1147e4100fec2ad252dab4ec0e1507bf6872b7063699fe793cc

data/README.md

@@ -47,7 +47,7 @@ slack_bot:  # this setting will be used for slack notice push
   channel: slack_channel
 ```
 
-+ config `config/database.yml` for development and production environment in redshift database  
++ config `config/red_access.yml` for development and production environment in redshift database  
 eg.
 
 ```YAML
@@ -75,14 +75,14 @@ Redata::Task.schema.config do
   # Example of declaring a global table
   table 'table_name'
   # This declaration means
-  #   query file: database/sources/table_name.red.sql
+  #   query file: red_query/sources/table_name.red.sql
   #   redshift table: table_name
   #   key used in command line: table_name
 
   # Example of declaring a global table with customizing options
   table 'new_table_name', :dir => 'dir', :file => 'query_file', :as => :alias
   # This declaration means
-  #   query file: database/sources/dir/query_file.red.sql
+  #   query file: red_query/sources/dir/query_file.red.sql
   #   redshift table: new_table_name
   #   key used in command line: alias
 
@@ -94,13 +94,13 @@ Redata::Task.schema.config do
   category :test_category do
     table 'test_table'
     # This declaration means
-    #   query file: database/sources/test_category/test_table.red.sql
+    #   query file: red_query/sources/test_category/test_table.red.sql
     #   redshift table: test_category_test_table
     #   key used in command line: test_category_test_table
 
     table 'test_table_oth', :dir => 'dir', :file => 'query_file_oth', :as => :alias_oth
     # This declaration means
-    #   query file: database/sources/dir/query_file_oth.red.sql
+    #   query file: red_query/sources/dir/query_file_oth.red.sql
     #   redshift table: test_category_test_table
     #   key used in command line: test_category_alias_oth
 
@@ -165,7 +165,7 @@ Usage: `redata [-options] [action] [object key] {stage}`
   - checkout --> export data in table/view of redshift into S3
   - deploy --> deploy data from S3 to local db or file
 + object key --> object declared in `config/relation.rb` will be create/delete/checkout/deploy
-+ stage --> when injecting data into mysql, there may be several stage declared in `config/database.yml{:deploy}` for same database, this could choose which stage to use.
++ stage --> when injecting data into mysql, there may be several stage declared in `config/red_access.yml{:deploy}` for same database, this could choose which stage to use.
 + options
   - -dir --> project directory, both absolute path and realtive path will be okay. default is current directory.
   - -e   --> environment: `production`, `development`, etc.
@@ -178,8 +178,8 @@ Usage: `redata [-options] [action] [object key] {stage}`
 
 Use adjust when you just want to run a query file without declaring in `config/relations.rb`  
 Usage: `adjust [-options] [database] [query file] {platform}`
-+ database   --> `redshift` or database declared in `config/database.yml{:deploy}`
-+ query file --> query file which will be run in `database/adjust/`, **without extends `.red.sql`**
++ database   --> `redshift` or database declared in `config/red_access.yml{:deploy}`
++ query file --> query file which will be run in `red_query/adjust/`, **without extends `.red.sql`**
 + platform   --> same to `redata`
 + options
   - -dir --> project directory, both absolute path and realtive path will be okay. default is current directory.

data/bin/adjust

@@ -4,7 +4,7 @@ require File.expand_path '../../lib/redata', __FILE__
 
 
 temp_config = OpenStruct.new
-temp_config.query_file = Redata::RED.root.join 'database', 'adjust', "#{Redata::RED.params[1]}.sql"
+temp_config.query_file = Redata::RED.root.join 'red_query', 'adjust', "#{Redata::RED.params[1]}.sql"
 temp_config.tmp_file_dir = Redata::RED.root.join "tmp", "adj_#{Redata::RED.params[0]}_#{Redata::RED.params[1]}"
 temp_config.tmp_exec_file = Redata::RED.root.join "tmp", "adj_#{Redata::RED.params[0]}_#{Redata::RED.params[1]}", "exec.sql"
 

data/lib/redata/config.rb

@@ -46,6 +46,10 @@ module Redata
 			@keep_tmp
 		end
 
+		def log_file
+			@root.join 'log', "#{@env}_redata.log"
+		end
+
 		def start_time
 			return @locals[:start_time] if @locals[:start_time]
 			if @is_append

data/lib/redata/database.rb

@@ -3,8 +3,8 @@ module Redata
 		REDSHIFT_PORT = 5439
 
 		def initialize
-			@config = YAML.load(ERB.new(File.read(RED.root.join 'config', 'database.yml')).result(binding))[RED.env]
-			Log.error! "ERROR: Database of #{RED.env} was not declared in config/database.yml" unless @config
+			@config = YAML.load(ERB.new(File.read(RED.root.join 'config', 'red_access.yml')).result(binding))[RED.env]
+			Log.error! "ERROR: Database of #{RED.env} was not declared in config/red_access.yml" unless @config
 
 			@ssh = Ssh.new
 		end
@@ -16,7 +16,7 @@ module Redata
 				@ssh.run_command "export PGPASSWORD='#{ENV['PGPASSWORD']}';#{cmd} -f ~/tmp/#{config.tmp_file_dir.basename}/exec.sql"
 				@ssh.remove_dir "~/tmp/#{config.tmp_file_dir.basename}"
 			else
-				system "#{cmd} -f #{config.tmp_exec_file}"
+				@ssh.local_command "#{cmd} -f #{config.tmp_exec_file}"
 			end
 		end
 
@@ -35,7 +35,7 @@ module Redata
 			else
 				Log.error! "ERROR: Export config of #{config.category} was not found" unless target_config
 			end
-			system cmd
+			@ssh.local_command cmd
 		end
 
 		def connect_mysql(query_file, category, stage)
@@ -46,7 +46,7 @@ module Redata
 			Log.error! "ERROR: Export config of #{config.category} for stage #{stage} was not found" unless target_config
 
 			cmd = "mysql #{make_mysql_cmd_params(target_config)} < #{query_file}"
-			system cmd
+			@ssh.local_command cmd
 		end
 
 		private

data/lib/redata/log.rb

@@ -1,20 +1,30 @@
 module Redata
 	class Log
 		def self.error!(msg)
-			puts "[#{RED.current_time}] #{msg.red}"
+			puts msg.red
+			self.file msg
 			exit 1
 		end
 
 		def self.warning(msg)
-			puts "[#{RED.current_time}] #{msg.yellow}"
+			puts msg.yellow
+			self.file msg
 		end
 
 		def self.action(msg)
-			puts "[#{RED.current_time}] #{msg.cyan}"
+			puts msg.cyan
+			self.file msg
 		end
 
 		def self.log(msg)
-			puts "[#{RED.current_time}] #{msg}"
+			puts msg
+			self.file msg
+		end
+
+		def self.file(msg)
+			File.open RED.log_file, 'a' do |f|
+				f.puts "[#{RED.current_time}] #{msg}"
+			end
 		end
 	end
 end

data/lib/redata/parser.rb

@@ -135,7 +135,7 @@ module Redata
 						Log.error! "QUERY ERROR: syntax error for load query: #{line}" if sub.empty? || name.empty?
 						
 						sub_file = in_file.parent.join "_#{sub}.red.sql"
-						sub_file = RED.root.join 'database', 'shared', "_#{sub}.rea.sql" unless sub_file.exist?
+						sub_file = RED.root.join 'red_query', 'shared', "_#{sub}.rea.sql" unless sub_file.exist?
 						sub_temp_tables = self.parse sub_file, out_file.dirname.join("#{name}.resql")
 						sub_temp_tables.each do |n|
 							temp_tables.push n unless temp_tables.include? n

data/lib/redata/relation.rb

@@ -18,7 +18,7 @@ module Redata
 		end
 
 		def query_file
-			query_file = RED.root.join 'database', 'sources'
+			query_file = RED.root.join 'red_query', 'sources'
 			query_file = query_file.join @dir if @dir
 			query_file = query_file.join "#{@file}.red.sql"
 			query_file

data/lib/redata/requires.rb

@@ -2,6 +2,7 @@
 require 'pathname'
 require 'yaml'
 require 'erb'
+require 'open3'
 
 # gem lib
 require 'json'

data/lib/redata/ssh.rb

@@ -18,15 +18,25 @@ module Redata
 		end
 
 		def upload_dir(dir)
-			system "scp -r -i #{@ssh['IdentityFile']} #{dir} #{@ssh['User']}@#{@ssh['HostName']}:~/tmp/"
+			`scp -r -i #{@ssh['IdentityFile']} #{dir} #{@ssh['User']}@#{@ssh['HostName']}:~/tmp/`
 		end
 
 		def run_command(cmd)
-			system "ssh -i #{@ssh['IdentityFile']} #{@ssh['User']}@#{@ssh['HostName']} \"#{cmd}\""
+			out, err, status = Open3.capture3 "ssh -i #{@ssh['IdentityFile']} #{@ssh['User']}@#{@ssh['HostName']} \"#{cmd}\""
+			puts err unless err.empty?
+			Log.file out unless out.empty?
+			Log.file err unless err.empty?
+		end
+
+		def local_command(cmd)
+			out, err, status = Open3.capture3 cmd
+			puts err unless err.empty?
+			Log.file out unless out.empty?
+			Log.file err unless err.empty?
 		end
 
 		def remove_dir(dir)
-			system "ssh -i #{@ssh['IdentityFile']} #{@ssh['User']}@#{@ssh['HostName']} \"rm -rf #{dir}\""
+			`ssh -i #{@ssh['IdentityFile']} #{@ssh['User']}@#{@ssh['HostName']} \"rm -rf #{dir}\"`
 		end
 	end
 end

data/lib/redata/version.rb

@@ -1,3 +1,3 @@
 module Redata
-  VERSION = "1.0.0"
+  VERSION = "1.1.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: redata
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.1
 platform: ruby
 authors:
 - goshan
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-02-23 00:00:00.000000000 Z
+date: 2017-02-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler