fluent-plugin-redshift 0.0.1 → 0.0.2

data/VERSION

@@ -1 +1 @@
-0.0.1
+0.0.2

data/lib/fluent/plugin/out_redshift.rb

@@ -4,6 +4,9 @@ module Fluent
 class RedshiftOutput < BufferedOutput
   Fluent::Plugin.register_output('redshift', self)
 
+  # ignore load table error. (invalid data format)
+  IGNORE_REDSHIFT_ERROR_REGEXP = /^ERROR:  Load into table '[^']+' failed\./
+
   def initialize
     super
     require 'aws-sdk'
@@ -34,6 +37,8 @@ class RedshiftOutput < BufferedOutput
   # file format
   config_param :file_type, :string, :default => nil  # json, tsv, csv
   config_param :delimiter, :string, :default => nil
+  # for debug
+  config_param :log_suffix, :string, :default => ''
 
   def configure(conf)
     super
@@ -48,8 +53,8 @@ class RedshiftOutput < BufferedOutput
       password:@redshift_password
     }
     @delimiter = determine_delimiter(@file_type) if @delimiter.nil? or @delimiter.empty?
-    $log.debug "redshift file_type:#{@file_type} delimiter:'#{@delimiter}'"
-    @copy_sql_template = "copy #{@redshift_tablename} from '%s' CREDENTIALS 'aws_access_key_id=#{@aws_key_id};aws_secret_access_key=%s' delimiter '#{@delimiter}' REMOVEQUOTES GZIP;"
+    $log.debug format_log("redshift file_type:#{@file_type} delimiter:'#{@delimiter}'")
+    @copy_sql_template = "copy #{@redshift_tablename} from '%s' CREDENTIALS 'aws_access_key_id=#{@aws_key_id};aws_secret_access_key=%s' delimiter '#{@delimiter}' GZIP TRUNCATECOLUMNS ESCAPE FILLRECORD ACCEPTANYDATE;"
   end
 
   def start
@@ -69,6 +74,8 @@ class RedshiftOutput < BufferedOutput
   end
 
   def write(chunk)
+    $log.debug format_log("start creating gz.")
+
     # create a gz file
     tmp = Tempfile.new("s3-")
     tmp = (json?) ? create_gz_file_from_json(tmp, chunk, @delimiter)
@@ -76,8 +83,8 @@ class RedshiftOutput < BufferedOutput
 
     # no data -> skip
     unless tmp
-      $log.debug "received no valid data. "
-      return
+      $log.debug format_log("received no valid data. ")
+      return false # for debug
     end
 
     # create a file path with time format
@@ -89,18 +96,25 @@ class RedshiftOutput < BufferedOutput
     # copy gz on s3 to redshift
     s3_uri = "s3://#{@s3_bucket}/#{s3path}"
     sql = @copy_sql_template % [s3_uri, @aws_sec_key]
-    $log.debug "start copying. s3_uri=#{s3_uri}"
+    $log.debug  format_log("start copying. s3_uri=#{s3_uri}")
     conn = nil
     begin
       conn = PG.connect(@db_conf)
       conn.exec(sql)
-      $log.info "completed copying to redshift. s3_uri=#{s3_uri}"
+      $log.info format_log("completed copying to redshift. s3_uri=#{s3_uri}")
     rescue PG::Error => e
-      $log.error "failed to copy data into redshift. sql=#{s3_uri}", :error=>e.to_s
-      raise e if e.result.nil? # retry if connection errors
+      $log.error format_log("failed to copy data into redshift. s3_uri=#{s3_uri}"), :error=>e.to_s
+      raise e unless e.to_s =~ IGNORE_REDSHIFT_ERROR_REGEXP
+      return false # for debug
     ensure
       conn.close rescue nil if conn
     end
+    true # for debug
+  end
+
+  protected
+  def format_log(message)
+    (@log_suffix and not @log_suffix.empty?) ? "#{message} #{@log_suffix}" : message
   end
 
   private
@@ -125,27 +139,24 @@ class RedshiftOutput < BufferedOutput
     if redshift_table_columns == nil
       raise "failed to fetch the redshift table definition."
     elsif redshift_table_columns.empty?
-      $log.warn "no table on redshift. table_name=#{@redshift_tablename}"
+      $log.warn format_log("no table on redshift. table_name=#{@redshift_tablename}")
       return nil
     end
 
     # convert json to tsv format text
-    table_texts = ""
-    chunk.msgpack_each do |record|
-      begin
-        table_texts << json_to_table_text(redshift_table_columns, record[@record_log_tag], delimiter)
-      rescue => e
-        $log.error "failed to create table text from json. text=(#{record[@record_log_tag]})", :error=>$!.to_s
-        $log.error_backtrace
-      end
-    end
-    return nil if table_texts.empty?
-
-    # create gz
     gzw = nil
     begin
       gzw = Zlib::GzipWriter.new(dst_file)
-      gzw.write(table_texts)
+      chunk.msgpack_each do |record|
+        begin
+          tsv_text = json_to_table_text(redshift_table_columns, record[@record_log_tag], delimiter)
+          gzw.write(tsv_text) if tsv_text and not tsv_text.empty?
+        rescue => e
+          $log.error format_log("failed to create table text from json. text=(#{record[@record_log_tag]})"), :error=>$!.to_s
+          $log.error_backtrace
+        end
+      end
+      return nil unless gzw.pos > 0
     ensure
       gzw.close rescue nil if gzw
     end
@@ -185,7 +196,7 @@ class RedshiftOutput < BufferedOutput
     begin
       json_obj = JSON.parse(json_text)
     rescue => e
-      $log.warn "failed to parse json. ", :error=>e.to_s
+      $log.warn format_log("failed to parse json. "), :error=>e.to_s
       return ""
     end
     return "" unless json_obj
@@ -198,19 +209,22 @@ class RedshiftOutput < BufferedOutput
       val.to_s unless val.nil?
     end
     if val_list.all?{|v| v.nil? or v.empty?}
-      $log.warn "no data match for table columns on redshift. json_text=#{json_text} table_columns=#{redshift_table_columns}"
+      $log.warn format_log("no data match for table columns on redshift. json_text=#{json_text} table_columns=#{redshift_table_columns}")
       return ""
     end
 
-    # generate tsv text
-    begin
-      CSV.generate(:col_sep=>delimiter, :quote_char => '"') do |row|
-        row << val_list # inlude new line
+    generate_line_with_delimiter(val_list, delimiter)
+  end
+
+  def generate_line_with_delimiter(val_list, delimiter)
+    val_list = val_list.collect do |val|
+      if val.nil? or val.empty?
+        ""
+      else
+        val.gsub(/\\/, "\\\\\\").gsub(/\t/, "\\\t").gsub(/\n/, "\\\n") # escape tab, newline and backslash
       end
-    rescue => e
-      $log.debug "failed to generate csv val_list:#{val_list} delimiter:(#{delimiter})"
-      raise e
     end
+    val_list.join(delimiter) + "\n"
   end
 
   def create_s3path(bucket, path)

data/test/plugin/test_out_redshift.rb

@@ -26,6 +26,7 @@ class RedshiftOutputTest < Test::Unit::TestCase
     redshift_tablename test_table
     buffer_type memory
     utc
+    log_suffix id:5 host:localhost
   ]
   CONFIG_CSV= %[
     #{CONFIG_BASE}
@@ -127,6 +128,10 @@ class RedshiftOutputTest < Test::Unit::TestCase
     assert_equal "pipe", d4.instance.file_type
     assert_equal "|", d4.instance.delimiter
   end
+  def test_configure_no_log_suffix
+    d = create_driver(CONFIG_CSV.gsub(/ *log_suffix *.+$/, ''))
+    assert_equal "", d.instance.log_suffix
+  end
 
   def emit_csv(d)
     d.emit(RECORD_CSV_A, DEFAULT_TIME)
@@ -231,30 +236,38 @@ class RedshiftOutputTest < Test::Unit::TestCase
     setup_mocks(%[val_a,val_b,val_c,val_d\nval_e,val_f,val_g,val_h\n])
     d_csv = create_driver
     emit_csv(d_csv)
-    d_csv.run
+    assert_equal true, d_csv.run
   end
 
   def test_write_with_json
     setup_mocks(%[val_a\tval_b\t\t\t\t\t\t\n\t\tval_c\tval_d\t\t\t\t\n])
     d_json = create_driver(CONFIG_JSON)
     emit_json(d_json)
-    d_json.run
+    assert_equal true, d_json.run
   end
 
   def test_write_with_json_hash_value
-    setup_mocks("val_a\t\"{\"\"foo\"\":\"\"var\"\"}\"\t\t\t\t\t\t\n\t\tval_c\tval_d\t\t\t\t\n")
+    setup_mocks("val_a\t{\"foo\":\"var\"}\t\t\t\t\t\t\n\t\tval_c\tval_d\t\t\t\t\n")
     d_json = create_driver(CONFIG_JSON)
     d_json.emit({"log" => %[{"key_a" : "val_a", "key_b" : {"foo" : "var"}}]} , DEFAULT_TIME)
     d_json.emit(RECORD_JSON_B, DEFAULT_TIME)
-    d_json.run
+    assert_equal true, d_json.run
   end
 
   def test_write_with_json_array_value
-    setup_mocks("val_a\t\"[\"\"foo\"\",\"\"var\"\"]\"\t\t\t\t\t\t\n\t\tval_c\tval_d\t\t\t\t\n")
+    setup_mocks("val_a\t[\"foo\",\"var\"]\t\t\t\t\t\t\n\t\tval_c\tval_d\t\t\t\t\n")
     d_json = create_driver(CONFIG_JSON)
     d_json.emit({"log" => %[{"key_a" : "val_a", "key_b" : ["foo", "var"]}]} , DEFAULT_TIME)
     d_json.emit(RECORD_JSON_B, DEFAULT_TIME)
-    d_json.run
+    assert_equal true, d_json.run
+  end
+
+  def test_write_with_json_including_tab_newline_quote
+    setup_mocks("val_a_with_\\\t_tab_\\\n_newline\tval_b_with_\\\\_quote\t\t\t\t\t\t\n\t\tval_c\tval_d\t\t\t\t\n")
+    d_json = create_driver(CONFIG_JSON)
+    d_json.emit({"log" => %[{"key_a" : "val_a_with_\\t_tab_\\n_newline", "key_b" : "val_b_with_\\\\_quote"}]} , DEFAULT_TIME)
+    d_json.emit(RECORD_JSON_B, DEFAULT_TIME)
+    assert_equal true, d_json.run
   end
 
   def test_write_with_json_no_data
@@ -262,7 +275,7 @@ class RedshiftOutputTest < Test::Unit::TestCase
     d_json = create_driver(CONFIG_JSON)
     d_json.emit("", DEFAULT_TIME)
     d_json.emit("", DEFAULT_TIME)
-    d_json.run
+    assert_equal false, d_json.run
   end
 
   def test_write_with_json_invalid_one_line
@@ -270,7 +283,7 @@ class RedshiftOutputTest < Test::Unit::TestCase
     d_json = create_driver(CONFIG_JSON)
     d_json.emit({"log" => %[}}]}, DEFAULT_TIME)
     d_json.emit(RECORD_JSON_B, DEFAULT_TIME)
-    d_json.run
+    assert_equal true, d_json.run
   end
 
   def test_write_with_json_no_available_data
@@ -278,7 +291,7 @@ class RedshiftOutputTest < Test::Unit::TestCase
     d_json = create_driver(CONFIG_JSON)
     d_json.emit(RECORD_JSON_A, DEFAULT_TIME)
     d_json.emit({"log" => %[{"key_o" : "val_o", "key_p" : "val_p"}]}, DEFAULT_TIME)
-    d_json.run
+    assert_equal true, d_json.run
   end
 
   def test_write_redshift_connection_error
@@ -300,14 +313,14 @@ class RedshiftOutputTest < Test::Unit::TestCase
     }
   end
 
-  def test_write_redshift_logic_error
+  def test_write_redshift_load_error
     PG::Error.module_eval { attr_accessor :result}
     def PG.connect(dbinfo)
       return Class.new do
         def initialize(return_keys=[]); end
         def exec(sql)
-          error = PG::Error.new("redshift logic error")
-          error.result = "logic error"
+          error = PG::Error.new("ERROR:  Load into table 'apache_log' failed.  Check 'stl_load_errors' system table for details.")
+          error.result = "ERROR:  Load into table 'apache_log' failed.  Check 'stl_load_errors' system table for details."
           raise error
         end
         def close; end
@@ -317,9 +330,7 @@ class RedshiftOutputTest < Test::Unit::TestCase
 
     d_csv = create_driver
     emit_csv(d_csv)
-    assert_nothing_raised {
-      d_csv.run
-    }
+    assert_equal false,  d_csv.run
   end
 
   def test_write_with_json_redshift_connection_error
@@ -356,9 +367,7 @@ class RedshiftOutputTest < Test::Unit::TestCase
 
     d_json = create_driver(CONFIG_JSON)
     emit_json(d_json)
-    assert_nothing_raised {
-      d_json.run
-    }
+    assert_equal false, d_json.run
   end
 
   def test_write_with_json_failed_to_get_columns
@@ -379,17 +388,4 @@ class RedshiftOutputTest < Test::Unit::TestCase
     }
   end
 
-  def test_write_with_json_failed_to_generate_tsv
-    flexmock(CSV).should_receive(:generate).with_any_args.
-        and_return {
-          raise "failed to generate tsv."
-        }
-    setup_s3_mock("")
-
-    d_json = create_driver(CONFIG_JSON)
-    emit_json(d_json)
-    assert_nothing_raised {
-      d_json.run
-    }
-  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-redshift
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-07 00:00:00.000000000 Z
+date: 2013-06-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd