patriot-aws 0.1.0 → 0.2.0

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    YTA4NTk4NTg3NjFjZGQ2NTc3OTBkYWQyNzViMjcyNTkzYzFiZDZhMA==
-  data.tar.gz: !binary |-
-    ZGMwYmJhNDkwZGIwYzhlYjdjZjM2NTQwZjNlNDljNmUyMDBmMzVlYg==
+SHA1:
+  metadata.gz: 26b16fcbab39dffb44c42a4badeb96236a388835
+  data.tar.gz: 7dcafac13fbbcfd9b701dc296697e99669058edd
 SHA512:
-  metadata.gz: !binary |-
-    MTA2NGJhMDEyNDNjNzhlOGQ1ZTY5MTI5Mzc5Y2FjZjBjNzljNzdmNmY2Zjc1
-    ODBlZmZjMDM1OWIxNDQxZjhlOGM0YzEyM2IzZWJiOGE1YjRmNTdhMzA2OGEx
-    NjQzZTdiMWI1YWYzM2EyODcwODg1ZTdlNWZkNGM4ZGMyNzJhNWE=
-  data.tar.gz: !binary |-
-    ODZmNTI4OWY4OWM1MzdjMDI0YzM0MWRhYjhmYWEwMmM4MGFhMjc5NTllMWRm
-    ODRhOTE2Zjg4NDNiYjY1NDY1NTg0ZTMwMWU2OTMyYmU1N2QxMTMzNjBkOGIz
-    OTkyOTU4ODgxMjBhNWNkMGM5OWY1MDhjZDAzZTM2OTkyZjRkMjE=
+  metadata.gz: 780e3b20dfa8cb4aec12935a29bbf378df59639ab5c5f981e7e343adce9ddcb2421c0bdcca2facf051e8912244b20b749329e12a9e859d52a22208c6f2462376
+  data.tar.gz: 450bd34d82813b575a2b66acff47d20c3fe3e7bc3bbd3e1bb19f11ccb88acb1b135d562c234396ce7355da2236d6c8687780f8eeea34346f30b30dc0bac99184

data/lib/patriot_aws/command.rb

@@ -1 +1,2 @@
 require 'patriot_aws/command/s3'
+require 'patriot_aws/command/redshift'

data/lib/patriot_aws/command/redshift.rb

@@ -0,0 +1,121 @@
+=begin
+
+sample_select.pbc
+--
+redshift {
+  name "select_from_redshift"
+  name_suffix _date_
+  inifile '/path/to//redshift.ini'
+  query 'select * from test_table'
+  options :with_header => true, :delimiter => "\t"
+}
+
+sample_copy.pbc
+--
+redshift {
+  name "s3_to_redshift"
+  name_suffix _date_
+  inifile '/Users/b06959/tmp/redshift.ini'
+  query <<-EOS
+    COPY #{schema}.#{table}
+    FROM '#{s3_path}'
+    ACCESS_KEY_ID '%{access_key_id}'
+    SECRET_ACCESS_KEY '%{secret_access_key}'
+    delimiter '\t'
+    gzip
+  EOS
+}
+
+$ cat redshift.ini
+[connection]
+host     = staging.xxxxxxxxxxxxx.ap-northeast-1.redshift.amazonaws.com
+user     = staging
+password = staging
+dbname   = staging
+port     = 5439
+
+[s3credentials]
+access_key_id     = #{access_key_id},
+secret_access_key = #{secret_access_key}
+=end
+
+require 'pg'
+
+module PatriotAWS
+  module Command
+    class RedshiftCommand < Patriot::Command::Base
+      declare_command_name :redshift
+      include PatriotAWS::Ext::AWS
+
+      command_attr :name, :name_suffix, :inifile, :options, :query
+
+      def job_id
+        job_id = "#{command_name}_#{@name}_#{@name_suffix}"
+        job_id
+      end
+
+      # @see Patriot::Command::Base#configure
+      def configure
+        @name_suffix ||= _date_
+        self
+      end
+
+      def execute
+        @logger.info 'start redshift query...'
+        @options ||= {}
+
+        ini = IniFile.load(@inifile)
+        raise Exception, 'inifile not found.' if ini.nil?
+        raise Exception, 'query is not set.'  if @query.nil?
+
+        _set_options
+
+        begin
+          # replace variables
+          if ini['s3credentials']
+            @query = @query % ini['s3credentials'].symbolize_keys
+          end
+
+          conn = PG::Connection.new(
+            ini['connection'].symbolize_keys
+          )
+          res   = conn.exec(@query)
+
+          output_arr = Array.new
+          if res then
+            res.each_with_index do |r, idx|
+              # print header
+              if @options[:with_header] && idx == 0
+                output_arr.push r.keys.join(@options[:delimiter])
+              end
+
+              output_arr.push r.values.join(@options[:delimiter])
+            end
+          end
+
+          puts output_arr.join("\n")
+        # rescue PGError => ex
+        #   # may cause PGError when connection setting is invalid
+        #   # e.g.
+        #   # PG::ConnectionBad -> could not translate host name "test" to address: nodename nor servname provided, or not known
+        #   print(ex.class," -> ",ex.message)
+        #
+        #   raise PGError
+        # rescue => ex
+        #   # Other Error process
+        #   print(ex.class," -> ",ex.message)
+        ensure
+          conn.close if conn
+        end
+      end
+
+      # @private
+      # set default option parameters
+      def _set_options
+        @options[:with_header]  = false if @options[:with_header].nil?
+        @options[:delimiter]    = "\t"  if @options[:delimiter].nil?
+      end
+      private :_set_options
+    end
+  end
+end

data/lib/patriot_aws/version.rb

@@ -1,4 +1,4 @@
 class VERSION
-  VERSION = "0.1.0"
+  VERSION = "0.2.0"
   PROJECT_NAME = "patriot-aws"
 end

metadata

@@ -1,41 +1,55 @@
 --- !ruby/object:Gem::Specification
 name: patriot-aws
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Takayuki Tanaka
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-04-25 00:00:00.000000000 Z
+date: 2017-06-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '2'
+- !ruby/object:Gem::Dependency
+  name: pg
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.4
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 0.18.4
 - !ruby/object:Gem::Dependency
   name: patriot-workflow-scheduler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.7'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '0.7'
 description: plugins for Patriot Workflow Scheduler, which deal with AWS such as S3.
@@ -48,6 +62,7 @@ files:
 - init.rb
 - lib/patriot_aws.rb
 - lib/patriot_aws/command.rb
+- lib/patriot_aws/command/redshift.rb
 - lib/patriot_aws/command/s3.rb
 - lib/patriot_aws/ext.rb
 - lib/patriot_aws/ext/aws.rb
@@ -62,19 +77,18 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: patriot-aws
-rubygems_version: 2.4.7
+rubygems_version: 2.6.12
 signing_key: 
 specification_version: 4
 summary: AWS plugin for Patriot Workflow Scheduler
 test_files: []
-has_rdoc: