fluent-plugin-redshift-out 0.3.0 → 0.4.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 5b20d3794db1fa89bcd3023c16a7521da561463d
+  data.tar.gz: c07149ac3e5cf6978b7ccd6b72becb7407274d88
+SHA512:
+  metadata.gz: fea913edcc8f21adf6e7c9b9068b9ae0830fc2813fe0b37640b868dcff1dbdb45fb55566f8a1e9c2e6acf59dfb3a1511105181941523811238aa98a36ddbc56b
+  data.tar.gz: 775d3b77816faa0c010e3f3eef54cec753f2ed12704bc1c89ed890c2db7464d2b206ff25a40e20ce9b63ca82e193fe10691e229c0c86e4ef987c9c33090c8a41

data/VERSION

@@ -1 +1 @@
-0.3.0
+0.4.0

data/fluent-plugin-redshift-out.gemspec

@@ -16,8 +16,8 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
-  gem.add_dependency "fluentd", "~> 0.10.0"
-  gem.add_dependency "aws-sdk", "~> 1.60"
+  gem.add_dependency "fluentd", [">= 0.10.0", "< 2"]
+  gem.add_dependency "aws-sdk-v1", ">= 1.6.3"
   gem.add_dependency "multi_json", "~> 1.10"
   gem.add_dependency "yajl-ruby", "~> 1.2"
   gem.add_dependency "pg", "~> 0.17.0"

data/lib/fluent/plugin/out_redshift-out.rb

@@ -4,12 +4,14 @@ module Fluent
 class RedshiftOutput < BufferedOutput
   Fluent::Plugin.register_output('redshift-out', self)
 
+  NULL_CHAR_FOR_COPY = "\\N"
+
   # ignore load table error. (invalid data format)
   IGNORE_REDSHIFT_ERROR_REGEXP = /^ERROR:  Load into table '[^']+' failed\./
 
   def initialize
     super
-    require 'aws-sdk'
+    require 'aws-sdk-v1'
     require 'zlib'
     require 'time'
     require 'tempfile'
@@ -22,8 +24,8 @@ class RedshiftOutput < BufferedOutput
 
   config_param :record_log_tag, :string, :default => 'log'
   # s3
-  config_param :aws_key_id, :string, :default => nil
-  config_param :aws_sec_key, :string, :default => nil
+  config_param :aws_key_id, :string, :secret => true, :default => nil
+  config_param :aws_sec_key, :string, :secret => true, :default => nil
   config_param :s3_bucket, :string
   config_param :s3_region, :string, :default => nil
   config_param :path, :string, :default => ""
@@ -34,7 +36,7 @@ class RedshiftOutput < BufferedOutput
   config_param :redshift_port, :integer, :default => 5439
   config_param :redshift_dbname, :string
   config_param :redshift_user, :string
-  config_param :redshift_password, :string
+  config_param :redshift_password, :string, :secret => true
   config_param :redshift_tablename, :string
   config_param :redshift_schemaname, :string, :default => nil
   config_param :redshift_copy_base_options, :string , :default => "FILLRECORD ACCEPTANYDATE TRUNCATECOLUMNS"
@@ -43,6 +45,8 @@ class RedshiftOutput < BufferedOutput
   # file format
   config_param :file_type, :string, :default => nil  # json, tsv, csv, msgpack
   config_param :delimiter, :string, :default => nil
+  # maintenance
+  config_param :maintenance_file_path, :string, :default => nil
   # for debug
   config_param :log_suffix, :string, :default => ''
 
@@ -61,6 +65,8 @@ class RedshiftOutput < BufferedOutput
     }
     @delimiter = determine_delimiter(@file_type) if @delimiter.nil? or @delimiter.empty?
     $log.debug format_log("redshift file_type:#{@file_type} delimiter:'#{@delimiter}'")
+    @table_name_with_schema = [@redshift_schemaname, @redshift_tablename].compact.join('.')
+    @maintenance_monitor = MaintenanceMonitor.new(@maintenance_file_path)
   end
 
   def start
@@ -74,6 +80,7 @@ class RedshiftOutput < BufferedOutput
     options[:region] = @s3_region if @s3_region
     @s3 = AWS::S3.new(options)
     @bucket = @s3.buckets[@s3_bucket]
+    @redshift_connection = RedshiftConnection.new(@db_conf)
   end
 
   def format(tag, time, record)
@@ -88,6 +95,7 @@ class RedshiftOutput < BufferedOutput
 
   def write(chunk)
     $log.debug format_log("start creating gz.")
+    @maintenance_monitor.check_maintenance!
 
     # create a gz file
     tmp = Tempfile.new("s3-")
@@ -117,7 +125,7 @@ class RedshiftOutput < BufferedOutput
     # copy gz on s3 to redshift
     s3_uri = "s3://#{@s3_bucket}/#{s3path}"
     credentials = @s3.client.credential_provider.credentials
-    sql = "copy #{table_name_with_schema} from '#{s3_uri}'"
+    sql = "copy #{@table_name_with_schema} from '#{s3_uri}'"
     sql += " CREDENTIALS 'aws_access_key_id=#{credentials[:access_key_id]};aws_secret_access_key=#{credentials[:secret_access_key]}"
     sql += ";token=#{credentials[:session_token]}" if credentials[:session_token]
     sql += "' delimiter '#{@delimiter}' GZIP ESCAPE #{@redshift_copy_base_options} #{@redshift_copy_options};"
@@ -125,25 +133,26 @@ class RedshiftOutput < BufferedOutput
     $log.debug format_log("start copying. s3_uri=#{s3_uri}")
 
     begin
-      conn = PG.connect(@db_conf)
-      conn.exec(sql)
+      @redshift_connection.exec(sql)
       $log.info format_log("completed copying to redshift. s3_uri=#{s3_uri}")
-    rescue PG::Error => e
-      $log.error format_log("failed to copy data into redshift. s3_uri=#{s3_uri}"), :error=>e.to_s
-      raise e unless e.to_s =~ IGNORE_REDSHIFT_ERROR_REGEXP
-      return false # for debug
-    ensure
-      conn.close rescue nil if conn
+    rescue RedshiftError => e
+      if e.to_s =~ IGNORE_REDSHIFT_ERROR_REGEXP
+        $log.error format_log("failed to copy data into redshift due to load error. s3_uri=#{s3_uri}"), :error=>e.to_s
+        return false # for debug
+      end
+      raise e
     end
     true # for debug
   end
 
   protected
+
   def format_log(message)
     (@log_suffix and not @log_suffix.empty?) ? "#{message} #{@log_suffix}" : message
   end
 
   private
+
   def json?
     @file_type == 'json'
   end
@@ -165,11 +174,11 @@ class RedshiftOutput < BufferedOutput
 
   def create_gz_file_from_structured_data(dst_file, chunk, delimiter)
     # fetch the table definition from redshift
-    redshift_table_columns = fetch_table_columns
+    redshift_table_columns = @redshift_connection.fetch_table_columns(@redshift_tablename, @redshift_schemaname)
     if redshift_table_columns == nil
       raise "failed to fetch the redshift table definition."
     elsif redshift_table_columns.empty?
-      $log.warn format_log("no table on redshift. table_name=#{table_name_with_schema}")
+      $log.warn format_log("no table on redshift. table_name=#{@table_name_with_schema}")
       return nil
     end
 
@@ -184,12 +193,7 @@ class RedshiftOutput < BufferedOutput
           tsv_text = hash_to_table_text(redshift_table_columns, hash, delimiter)
           gzw.write(tsv_text) if tsv_text and not tsv_text.empty?
         rescue => e
-          if json?
-            $log.error format_log("failed to create table text from json. text=(#{record[@record_log_tag]})"), :error=>e.to_s
-          else
-            $log.error format_log("failed to create table text from msgpack. text=(#{record[@record_log_tag]})"), :error=>e.to_s
-          end
-
+          $log.error format_log("failed to create table text from #{@file_type}. text=(#{record[@record_log_tag]})"), :error=>e.to_s
           $log.error_backtrace
         end
       end
@@ -211,27 +215,6 @@ class RedshiftOutput < BufferedOutput
     end
   end
 
-  def fetch_table_columns
-    begin
-      columns = nil
-      conn = PG.connect(@db_conf)
-      conn.exec(fetch_columns_sql_with_schema) do |result|
-        columns = result.collect{|row| row['column_name']}
-      end
-      columns
-    ensure
-      conn.close rescue nil if conn
-    end
-  end
-
-  def fetch_columns_sql_with_schema
-    @fetch_columns_sql ||= if @redshift_schemaname
-                             "select column_name from INFORMATION_SCHEMA.COLUMNS where table_schema = '#{@redshift_schemaname}' and table_name = '#{@redshift_tablename}' order by ordinal_position;"
-                           else
-                             "select column_name from INFORMATION_SCHEMA.COLUMNS where table_name = '#{@redshift_tablename}' order by ordinal_position;"
-                           end
-  end
-
   def json_to_hash(json_text)
     return nil if json_text.to_s.empty?
 
@@ -245,18 +228,9 @@ class RedshiftOutput < BufferedOutput
     return "" unless hash
 
     # extract values from hash
-    val_list = redshift_table_columns.collect do |cn|
-      val = hash[cn]
-      val = MultiJson.dump(val) if val.kind_of?(Hash) or val.kind_of?(Array)
-
-      if val.to_s.empty?
-        nil
-      else
-        val.to_s
-      end
-    end
+    val_list = redshift_table_columns.collect {|cn| hash[cn]}
 
-    if val_list.all?{|v| v.nil? or v.empty?}
+    if val_list.all?{|v| v.nil?}
       $log.warn format_log("no data match for table columns on redshift. data=#{hash} table_columns=#{redshift_table_columns}")
       return ""
     end
@@ -265,14 +239,22 @@ class RedshiftOutput < BufferedOutput
   end
 
   def generate_line_with_delimiter(val_list, delimiter)
-    val_list = val_list.collect do |val|
-      if val.nil? or val.empty?
-        ""
+    val_list.collect do |val|
+      case val
+      when nil
+        NULL_CHAR_FOR_COPY
+      when ''
+        ''
+      when Hash, Array
+        escape_text_for_copy(MultiJson.dump(val))
       else
-        val.gsub(/\\/, "\\\\\\").gsub(/\t/, "\\\t").gsub(/\n/, "\\\n") # escape tab, newline and backslash
+        escape_text_for_copy(val.to_s)
       end
-    end
-    val_list.join(delimiter) + "\n"
+    end.join(delimiter) + "\n"
+  end
+
+  def escape_text_for_copy(val)
+    val.gsub(/\\|\t|\n/, {"\\" => "\\\\", "\t" => "\\\t", "\n" => "\\\n"})  # escape tab, newline and backslash
   end
 
   def create_s3path(bucket, path)
@@ -286,12 +268,124 @@ class RedshiftOutput < BufferedOutput
     s3path
   end
 
-  def table_name_with_schema
-    @table_name_with_schema ||= if @redshift_schemaname
-                                  "#{@redshift_schemaname}.#{@redshift_tablename}"
-                                else
-                                  @redshift_tablename
-                                end
+  class RedshiftError < StandardError
+    def initialize(msg)
+      case msg
+      when PG::Error
+        @pg_error = msg
+        super(msg.to_s)
+        set_backtrace(msg.backtrace)
+      else
+        super
+      end
+    end
+
+    attr_accessor :pg_error
+  end
+
+  class RedshiftConnection
+    REDSHIFT_CONNECT_TIMEOUT = 10.0  # 10sec
+
+    def initialize(db_conf)
+      @db_conf = db_conf
+      @connection = nil
+    end
+
+    attr_reader :db_conf
+
+    def fetch_table_columns(table_name, schema_name)
+      columns = nil
+      exec(fetch_columns_sql(table_name, schema_name)) do |result|
+        columns = result.collect{|row| row['column_name']}
+      end
+      columns
+    end
+
+    def exec(sql, &block)
+      conn = @connection
+      conn = create_redshift_connection if conn.nil?
+      if block
+        conn.exec(sql) {|result| block.call(result)}
+      else
+        conn.exec(sql)
+      end
+    rescue PG::Error => e
+      raise RedshiftError.new(e)
+    ensure
+      conn.close if conn && @connection.nil?
+    end
+
+    def connect_start
+      @connection = create_redshift_connection
+    end
+
+    def close
+      @connection.close rescue nil if @connection
+      @connection = nil
+    end
+
+    private
+
+    def create_redshift_connection
+      hostaddr = IPSocket.getaddress(db_conf[:host])
+      db_conf[:hostaddr] = hostaddr
+
+      conn = PG::Connection.connect_start(db_conf)
+      raise RedshiftError.new("Unable to create a new connection.") unless conn
+      if conn.status == PG::CONNECTION_BAD
+        raise RedshiftError.new("Connection failed: %s" % [ conn.error_message ])
+      end
+
+      socket = conn.socket_io
+      poll_status = PG::PGRES_POLLING_WRITING
+      until poll_status == PG::PGRES_POLLING_OK || poll_status == PG::PGRES_POLLING_FAILED
+        case poll_status
+        when PG::PGRES_POLLING_READING
+          IO.select([socket], nil, nil, REDSHIFT_CONNECT_TIMEOUT) or
+            raise RedshiftError.new("Asynchronous connection timed out!(READING)")
+        when PG::PGRES_POLLING_WRITING
+          IO.select(nil, [socket], nil, REDSHIFT_CONNECT_TIMEOUT) or
+            raise RedshiftError.new("Asynchronous connection timed out!(WRITING)")
+        end
+        poll_status = conn.connect_poll
+      end
+
+      unless conn.status == PG::CONNECTION_OK
+        raise RedshiftError, ("Connect failed: %s" % [conn.error_message.to_s.lines.uniq.join(" ")])
+      end
+
+      conn
+    rescue => e
+      conn.close rescue nil if conn
+      raise RedshiftError.new(e) if e.kind_of?(PG::Error)
+      raise e
+    end
+
+    def fetch_columns_sql(table_name, schema_name = nil)
+      sql = "select column_name from INFORMATION_SCHEMA.COLUMNS where table_name = '#{table_name}'"
+      sql << " and table_schema = '#{schema_name}'" if schema_name
+      sql << " order by ordinal_position;"
+      sql
+    end
+  end
+
+  class MaintenanceError < StandardError
+  end
+
+  class MaintenanceMonitor
+    def initialize(maintenance_file_path)
+      @file_path = maintenance_file_path
+    end
+
+    def in_maintenance?
+      !!(@file_path && File.exists?(@file_path))
+    end
+
+    def check_maintenance!
+      if in_maintenance?
+        raise MaintenanceError.new("Service is in maintenance mode - maintenance_file_path:#{@file_path}")
+      end
+    end
   end
 end
 

data/test/plugin/test_out_redshift.rb

@@ -1,19 +1,22 @@
 require 'test_helper'
 
 require 'fluent/test'
-require 'fluent/plugin/out_redshift'
+require 'fluent/plugin/out_redshift-out'
 require 'flexmock/test_unit'
 require 'zlib'
 
 
 class RedshiftOutputTest < Test::Unit::TestCase
   def setup
-    require 'aws-sdk'
+    require 'aws-sdk-v1'
     require 'pg'
     require 'csv'
     Fluent::Test.setup
+    PG::Error.module_eval { attr_accessor :result}
   end
 
+  MAINTENANCE_FILE_PATH_FOR_TEST = "/tmp/fluentd_redshift_plugin_test_maintenance"
+
   CONFIG_BASE= %[
     aws_key_id test_key_id
     aws_sec_key test_sec_key
@@ -27,6 +30,7 @@ class RedshiftOutputTest < Test::Unit::TestCase
     buffer_type memory
     utc
     log_suffix id:5 host:localhost
+    maintenance_file_path #{MAINTENANCE_FILE_PATH_FOR_TEST}
   ]
   CONFIG_CSV= %[
     #{CONFIG_BASE}
@@ -106,6 +110,7 @@ class RedshiftOutputTest < Test::Unit::TestCase
     assert_equal "csv", d.instance.file_type
     assert_equal ",", d.instance.delimiter
     assert_equal true, d.instance.utc
+    assert_equal MAINTENANCE_FILE_PATH_FOR_TEST, d.instance.maintenance_file_path
   end
   def test_configure_with_schemaname
     d = create_driver(CONFIG_JSON_WITH_SCHEMA)
@@ -211,54 +216,45 @@ class RedshiftOutputTest < Test::Unit::TestCase
     d_msgpack.run
   end
 
-  class PGConnectionMock
-    def initialize(options = {})
-      @return_keys = options[:return_keys] || ['key_a', 'key_b', 'key_c', 'key_d', 'key_e', 'key_f', 'key_g', 'key_h']
-      @target_schema = options[:schemaname] || nil
-      @target_table = options[:tablename] || 'test_table'
-    end
+  def setup_redshift_connection_mock(options = {})
+    options ||= {}
+    column_names = options[:column_names] || ['key_a', 'key_b', 'key_c', 'key_d', 'key_e', 'key_f', 'key_g', 'key_h']
+    schema_name = options[:schema_name]
+    table_name = options[:table_name] || 'test_table'
+    exec_sql_proc = options[:exec_sql_proc]
 
-    def expected_column_list_query
-      if @target_schema
-        /\Aselect column_name from INFORMATION_SCHEMA.COLUMNS where table_schema = '#{@target_schema}' and table_name = '#{@target_table}'/
+    column_list_query_regex =
+      if schema_name
+        /\Aselect column_name from INFORMATION_SCHEMA.COLUMNS where table_name = '#{table_name}' and table_schema = '#{schema_name}'/
       else
-        /\Aselect column_name from INFORMATION_SCHEMA.COLUMNS where table_name = '#{@target_table}'/
+        /\Aselect column_name from INFORMATION_SCHEMA.COLUMNS where table_name = '#{table_name}'/
       end
-    end
-
-    def expected_copy_query
-      if @target_schema
-        /\Acopy #{@target_schema}.#{@target_table} from/
+    copy_query_regex =
+      if schema_name
+        /\Acopy #{schema_name}.#{table_name} from/
       else
-        /\Acopy #{@target_table} from/
+        /\Acopy #{table_name} from/
       end
-    end
 
-    def exec(sql, &block)
-      if block_given?
-        if sql =~ expected_column_list_query
-          yield @return_keys.collect{|key| {'column_name' => key}}
+    flexmock(Fluent::RedshiftOutput::RedshiftConnection).new_instances do |conn|
+      conn.should_receive(:exec).and_return do |sql, block|
+        if exec_sql_proc
+          exec_sql_proc.call(sql, block)
+        elsif block
+          if sql =~ column_list_query_regex
+            block.call column_names.collect{|key| {'column_name' => key}}
+          else
+            block.call []
+          end
         else
-          yield []
-        end
-      else
-        unless sql =~ expected_copy_query
-          error = PG::Error.new("ERROR:  Load into table '#{@target_table}' failed.  Check 'stl_load_errors' system table for details.")
-          error.result = "ERROR:  Load into table '#{@target_table}' failed.  Check 'stl_load_errors' system table for details."
-          raise error
+          unless sql =~ copy_query_regex
+            error = PG::Error.new("ERROR:  Load into table '#{@target_table}' failed.  Check 'stl_load_errors' system table for details.")
+            error.result = "ERROR:  Load into table '#{@target_table}' failed.  Check 'stl_load_errors' system table for details."
+            raise Fluent::RedshiftOutput::RedshiftError.new(error)
+          end
         end
       end
     end
-
-    def close
-    end
-  end
-
-  def setup_pg_mock
-    # create mock of PG
-    def PG.connect(dbinfo)
-      return PGConnectionMock.new
-    end
   end
 
   def setup_s3_mock(expected_data)
@@ -305,9 +301,10 @@ class RedshiftOutputTest < Test::Unit::TestCase
     flexmock(Tempfile).new_instances.should_receive(:close!).at_least.once
   end
 
-  def setup_mocks(expected_data)
-    setup_pg_mock
-    setup_s3_mock(expected_data) end
+  def setup_mocks(expected_data, options = {})
+    setup_redshift_connection_mock(options)
+    setup_s3_mock(expected_data)
+  end
 
   def test_write_with_csv
     setup_mocks(%[val_a,val_b,val_c,val_d\nval_e,val_f,val_g,val_h\n])
@@ -318,7 +315,7 @@ class RedshiftOutputTest < Test::Unit::TestCase
   end
 
   def test_write_with_json
-    setup_mocks(%[val_a\tval_b\t\t\t\t\t\t\n\t\tval_c\tval_d\t\t\t\t\n])
+    setup_mocks(%[val_a\tval_b\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\n\\N\t\\N\tval_c\tval_d\t\\N\t\\N\t\\N\t\\N\n])
     setup_tempfile_mock_to_be_closed
     d_json = create_driver(CONFIG_JSON)
     emit_json(d_json)
@@ -326,7 +323,7 @@ class RedshiftOutputTest < Test::Unit::TestCase
   end
 
   def test_write_with_json_hash_value
-    setup_mocks("val_a\t{\"foo\":\"var\"}\t\t\t\t\t\t\n\t\tval_c\tval_d\t\t\t\t\n")
+    setup_mocks("val_a\t{\"foo\":\"var\"}\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\n\\N\t\\N\tval_c\tval_d\t\\N\t\\N\t\\N\t\\N\n")
     d_json = create_driver(CONFIG_JSON)
     d_json.emit({"log" => %[{"key_a" : "val_a", "key_b" : {"foo" : "var"}}]} , DEFAULT_TIME)
     d_json.emit(RECORD_JSON_B, DEFAULT_TIME)
@@ -334,7 +331,7 @@ class RedshiftOutputTest < Test::Unit::TestCase
   end
 
   def test_write_with_json_array_value
-    setup_mocks("val_a\t[\"foo\",\"var\"]\t\t\t\t\t\t\n\t\tval_c\tval_d\t\t\t\t\n")
+    setup_mocks("val_a\t[\"foo\",\"var\"]\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\n\\N\t\\N\tval_c\tval_d\t\\N\t\\N\t\\N\t\\N\n")
     d_json = create_driver(CONFIG_JSON)
     d_json.emit({"log" => %[{"key_a" : "val_a", "key_b" : ["foo", "var"]}]} , DEFAULT_TIME)
     d_json.emit(RECORD_JSON_B, DEFAULT_TIME)
@@ -342,13 +339,20 @@ class RedshiftOutputTest < Test::Unit::TestCase
   end
 
   def test_write_with_json_including_tab_newline_quote
-    setup_mocks("val_a_with_\\\t_tab_\\\n_newline\tval_b_with_\\\\_quote\t\t\t\t\t\t\n\t\tval_c\tval_d\t\t\t\t\n")
+    setup_mocks("val_a_with_\\\t_tab_\\\n_newline\tval_b_with_\\\\_quote\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\n\\N\t\\N\tval_c\tval_d\t\\N\t\\N\t\\N\t\\N\n")
     d_json = create_driver(CONFIG_JSON)
     d_json.emit({"log" => %[{"key_a" : "val_a_with_\\t_tab_\\n_newline", "key_b" : "val_b_with_\\\\_quote"}]} , DEFAULT_TIME)
     d_json.emit(RECORD_JSON_B, DEFAULT_TIME)
     assert_equal true, d_json.run
   end
 
+  def test_write_with_json_empty_text_value
+    setup_mocks(%[val_a\t\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\n])
+    d_json = create_driver(CONFIG_JSON)
+    d_json.emit({"log" => %[{"key_a" : "val_a", "key_b" : ""}]} , DEFAULT_TIME)
+    assert_equal true, d_json.run
+  end
+
   def test_write_with_json_no_data
     setup_mocks("")
     d_json = create_driver(CONFIG_JSON)
@@ -358,7 +362,7 @@ class RedshiftOutputTest < Test::Unit::TestCase
   end
 
   def test_write_with_json_invalid_one_line
-    setup_mocks(%[\t\tval_c\tval_d\t\t\t\t\n])
+    setup_mocks(%[\\N\t\\N\tval_c\tval_d\t\\N\t\\N\t\\N\t\\N\n])
     d_json = create_driver(CONFIG_JSON)
     d_json.emit({"log" => %[}}]}, DEFAULT_TIME)
     d_json.emit(RECORD_JSON_B, DEFAULT_TIME)
@@ -366,7 +370,7 @@ class RedshiftOutputTest < Test::Unit::TestCase
   end
 
   def test_write_with_json_no_available_data
-    setup_mocks(%[val_a\tval_b\t\t\t\t\t\t\n])
+    setup_mocks(%[val_a\tval_b\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\n])
     d_json = create_driver(CONFIG_JSON)
     d_json.emit(RECORD_JSON_A, DEFAULT_TIME)
     d_json.emit({"log" => %[{"key_o" : "val_o", "key_p" : "val_p"}]}, DEFAULT_TIME)
@@ -374,14 +378,14 @@ class RedshiftOutputTest < Test::Unit::TestCase
   end
 
   def test_write_with_msgpack
-    setup_mocks(%[val_a\tval_b\t\t\t\t\t\t\n\t\tval_c\tval_d\t\t\t\t\n])
+    setup_mocks(%[val_a\tval_b\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\n\\N\t\\N\tval_c\tval_d\t\\N\t\\N\t\\N\t\\N\n])
     d_msgpack = create_driver(CONFIG_MSGPACK)
     emit_msgpack(d_msgpack)
     assert_equal true, d_msgpack.run
   end
 
   def test_write_with_msgpack_hash_value
-    setup_mocks("val_a\t{\"foo\":\"var\"}\t\t\t\t\t\t\n\t\tval_c\tval_d\t\t\t\t\n")
+    setup_mocks("val_a\t{\"foo\":\"var\"}\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\n\\N\t\\N\tval_c\tval_d\t\\N\t\\N\t\\N\t\\N\n")
     d_msgpack = create_driver(CONFIG_MSGPACK)
     d_msgpack.emit({"key_a" => "val_a", "key_b" => {"foo" => "var"}} , DEFAULT_TIME)
     d_msgpack.emit(RECORD_MSGPACK_B, DEFAULT_TIME)
@@ -389,7 +393,7 @@ class RedshiftOutputTest < Test::Unit::TestCase
   end
 
   def test_write_with_msgpack_array_value
-    setup_mocks("val_a\t[\"foo\",\"var\"]\t\t\t\t\t\t\n\t\tval_c\tval_d\t\t\t\t\n")
+    setup_mocks("val_a\t[\"foo\",\"var\"]\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\n\\N\t\\N\tval_c\tval_d\t\\N\t\\N\t\\N\t\\N\n")
     d_msgpack = create_driver(CONFIG_MSGPACK)
     d_msgpack.emit({"key_a" => "val_a", "key_b" => ["foo", "var"]} , DEFAULT_TIME)
     d_msgpack.emit(RECORD_MSGPACK_B, DEFAULT_TIME)
@@ -397,7 +401,7 @@ class RedshiftOutputTest < Test::Unit::TestCase
   end
 
   def test_write_with_msgpack_including_tab_newline_quote
-    setup_mocks("val_a_with_\\\t_tab_\\\n_newline\tval_b_with_\\\\_quote\t\t\t\t\t\t\n\t\tval_c\tval_d\t\t\t\t\n")
+    setup_mocks("val_a_with_\\\t_tab_\\\n_newline\tval_b_with_\\\\_quote\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\n\\N\t\\N\tval_c\tval_d\t\\N\t\\N\t\\N\t\\N\n")
     d_msgpack = create_driver(CONFIG_MSGPACK)
     d_msgpack.emit({"key_a" => "val_a_with_\t_tab_\n_newline", "key_b" => "val_b_with_\\_quote"} , DEFAULT_TIME)
     d_msgpack.emit(RECORD_MSGPACK_B, DEFAULT_TIME)
@@ -413,7 +417,7 @@ class RedshiftOutputTest < Test::Unit::TestCase
   end
 
   def test_write_with_msgpack_no_available_data
-    setup_mocks(%[val_a\tval_b\t\t\t\t\t\t\n])
+    setup_mocks(%[val_a\tval_b\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\n])
     d_msgpack = create_driver(CONFIG_MSGPACK)
     d_msgpack.emit(RECORD_MSGPACK_A, DEFAULT_TIME)
     d_msgpack.emit({"key_o" => "val_o", "key_p" => "val_p"}, DEFAULT_TIME)
@@ -421,38 +425,21 @@ class RedshiftOutputTest < Test::Unit::TestCase
   end
 
   def test_write_redshift_connection_error
-    def PG.connect(dbinfo)
-      return Class.new do
-        def initialize(return_keys=[]); end
-        def exec(sql)
-          raise PG::Error, "redshift connection error"
-        end
-        def close; end
-      end.new
-    end
-    setup_s3_mock(%[val_a,val_b,val_c,val_d\nval_e,val_f,val_g,val_h\n])
-
+    setup_mocks(%[val_a,val_b,val_c,val_d\nval_e,val_f,val_g,val_h\n],
+      exec_sql_proc: Proc.new {|sql, block| raise Fluent::RedshiftOutput::RedshiftError, "redshift connection error" })
     d_csv = create_driver
     emit_csv(d_csv)
-    assert_raise(PG::Error) {
+    assert_raise(Fluent::RedshiftOutput::RedshiftError) {
       d_csv.run
     }
   end
 
   def test_write_redshift_load_error
-    PG::Error.module_eval { attr_accessor :result}
-    def PG.connect(dbinfo)
-      return Class.new do
-        def initialize(return_keys=[]); end
-        def exec(sql)
-          error = PG::Error.new("ERROR:  Load into table 'apache_log' failed.  Check 'stl_load_errors' system table for details.")
-          error.result = "ERROR:  Load into table 'apache_log' failed.  Check 'stl_load_errors' system table for details."
-          raise error
-        end
-        def close; end
-      end.new
-    end
-    setup_s3_mock(%[val_a,val_b,val_c,val_d\nval_e,val_f,val_g,val_h\n])
+    setup_mocks(%[val_a,val_b,val_c,val_d\nval_e,val_f,val_g,val_h\n],
+      exec_sql_proc: Proc.new {|sql, block|
+      msg = "ERROR:  Load into table 'apache_log' failed.  Check 'stl_load_errors' system table for details."
+      raise Fluent::RedshiftOutput::RedshiftError.new(msg)
+    })
 
     d_csv = create_driver
     emit_csv(d_csv)
@@ -460,36 +447,19 @@ class RedshiftOutputTest < Test::Unit::TestCase
   end
 
   def test_write_with_json_redshift_connection_error
-    def PG.connect(dbinfo)
-      return Class.new do
-        def initialize(return_keys=[]); end
-        def exec(sql, &block)
-          error = PG::Error.new("redshift connection error")
-          raise error
-        end
-        def close; end
-      end.new
-    end
-    setup_s3_mock(%[val_a,val_b,val_c,val_d\nval_e,val_f,val_g,val_h\n])
+    setup_mocks(%[val_a,val_b,val_c,val_d\nval_e,val_f,val_g,val_h\n],
+      exec_sql_proc: Proc.new {|sql, block| raise Fluent::RedshiftOutput::RedshiftError.new("redshift connection error")})
 
     d_json = create_driver(CONFIG_JSON)
     emit_json(d_json)
-    assert_raise(PG::Error) {
+    assert_raise(Fluent::RedshiftOutput::RedshiftError) {
       d_json.run
     }
   end
 
   def test_write_with_json_no_table_on_redshift
-    def PG.connect(dbinfo)
-      return Class.new do
-        def initialize(return_keys=[]); end
-        def exec(sql, &block)
-          yield [] if block_given?
-        end
-        def close; end
-      end.new
-    end
-    setup_s3_mock(%[val_a,val_b,val_c,val_d\nval_e,val_f,val_g,val_h\n])
+    setup_mocks(%[val_a,val_b,val_c,val_d\nval_e,val_f,val_g,val_h\n],
+                exec_sql_proc: Proc.new {|sql, block| block.call [] if block })
 
     d_json = create_driver(CONFIG_JSON)
     emit_json(d_json)
@@ -497,15 +467,7 @@ class RedshiftOutputTest < Test::Unit::TestCase
   end
 
   def test_write_with_json_failed_to_get_columns
-    def PG.connect(dbinfo)
-      return Class.new do
-        def initialize(return_keys=[]); end
-        def exec(sql, &block)
-        end
-        def close; end
-      end.new
-    end
-    setup_s3_mock("")
+    setup_mocks("", exec_sql_proc: Proc.new {|sql, block| nil})
 
     d_json = create_driver(CONFIG_JSON)
     emit_json(d_json)
@@ -515,12 +477,21 @@ class RedshiftOutputTest < Test::Unit::TestCase
   end
 
   def test_write_with_json_fetch_column_with_schema
-    def PG.connect(dbinfo)
-      return PGConnectionMock.new(:schemaname => 'test_schema')
-    end
-    setup_s3_mock(%[val_a\tval_b\t\t\t\t\t\t\n\t\tval_c\tval_d\t\t\t\t\n])
+    setup_mocks(%[val_a\tval_b\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\n\\N\t\\N\tval_c\tval_d\t\\N\t\\N\t\\N\t\\N\n],
+               schema_name: 'test_schema')
     d_json = create_driver(CONFIG_JSON_WITH_SCHEMA)
     emit_json(d_json)
     assert_equal true, d_json.run
   end
+
+  def test_maintenance_mode
+    flexmock(File).should_receive(:exists?).with(MAINTENANCE_FILE_PATH_FOR_TEST).and_return(true)
+
+    d_json = create_driver(CONFIG_JSON)
+    emit_json(d_json)
+    assert_raise(Fluent::RedshiftOutput::MaintenanceError,
+                 "Service is in maintenance mode - maintenance_file_path:#{MAINTENANCE_FILE_PATH_FOR_TEST}") {
+      d_json.run
+    }
+  end
 end

metadata

@@ -1,142 +1,131 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-redshift-out
 version: !ruby/object:Gem::Version
-  version: 0.3.0
-  prerelease: 
+  version: 0.4.0
 platform: ruby
 authors:
 - Naohiro Sakuma
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-01-06 00:00:00.000000000 Z
+date: 2015-12-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.10.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.10.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
 - !ruby/object:Gem::Dependency
-  name: aws-sdk
+  name: aws-sdk-v1
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.60'
+        version: 1.6.3
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.60'
+        version: 1.6.3
 - !ruby/object:Gem::Dependency
   name: multi_json
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.10'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.10'
 - !ruby/object:Gem::Dependency
   name: yajl-ruby
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.2'
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.17.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.17.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.5.4
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.5.4
 - !ruby/object:Gem::Dependency
   name: flexmock
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.3.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: 1.3.1
 description: Amazon Redshift output plugin for Fluentd
@@ -146,7 +135,7 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
+- ".gitignore"
 - Gemfile
 - README.md
 - Rakefile
@@ -157,27 +146,26 @@ files:
 - test/test_helper.rb
 homepage: https://github.com/sakuma/fluent-plugin-redshift-out
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.4.5.1
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Amazon Redshift output plugin for Fluentd
 test_files:
 - test/plugin/test_out_redshift.rb