fluent-plugin-redshift 0.0.6 → 0.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ae21e38c3e70d5c36c0c52bcf3e38183756e4534
-  data.tar.gz: a2ebf5bc56f51b9c5f4dbd55fcf07499d9bc9bd8
+  metadata.gz: 208e99381f5503be7e6af6fa3ad54fc2cbeb9bea
+  data.tar.gz: ed2707d372c126d420fd907bc409b5d0fede132d
 SHA512:
-  metadata.gz: 064a44a6ed9086b1aef44eded321e984a9ccb8afc530e46b03ad99a77b2a2b384c24900ceea220f92c96871c1db18c9503910aff50ccc2b36d042e3d6077d80d
-  data.tar.gz: c85eb3832be2f660f8974b535e95430a51c2a13556a97d3f417eec816a05eb78c015ba079080847a46f48b3f6350cbddd3a06bae12464bcdcba4a26776e28390
+  metadata.gz: f8f93c9b5bb3ba860a3c6900c1889453f9417a36bce39def88e82b5ca4bc7dd8a936b6cf805d246317daafd9f2cab168fa7d890a61bbad7b719b14c40321954b
+  data.tar.gz: d19b4d4751c71d293493cce89cf93224115c762a32a00e363ce774ffdb40798598f656a6e7de126e0b13012011c311a52bef86804f785ec490b52ad67755876b

data/README.md

@@ -39,6 +39,9 @@ Format:
         buffer_path /var/log/fluent/redshift
         flush_interval 15m
         buffer_chunk_limit 1g
+
+        # maintenance
+        maintenance_file_path /path/to/maintenance_file
     </match>
 
 Example (watch and upload json formatted apache log):
@@ -76,6 +79,9 @@ Example (watch and upload json formatted apache log):
         buffer_path /var/log/fluent/redshift
         flush_interval 15m
         buffer_chunk_limit 1g
+
+        # maintenance
+        maintenance_file_path /tmp/fluentd_redshift_maintenance
     </match>
 
 + `type` (required) : The value must be `redshift`.
@@ -129,6 +135,8 @@ Example (watch and upload json formatted apache log):
 
 + `utc` : utc time zone. This parameter affects `timestamp_key_format`.
 
++ `maintenance_file_path` : path of maintenance file. plugin skip processing and keep retrying during a file existing in this file path. To avoid data loss due to too many retries caused by long mainenance, setting `retry_limit` and `retry_wait` is recommended.
+
 ## Logging examples
 ```ruby
 # examples by fluent-logger

data/VERSION

@@ -1 +1 @@
-0.0.6
+0.1.0

data/fluent-plugin-redshift.gemspec

@@ -16,7 +16,7 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
-  gem.add_dependency "fluentd", "~> 0.10.0"
+  gem.add_dependency "fluentd", [">= 0.10.0", "< 2"]
   gem.add_dependency "aws-sdk-v1", ">= 1.6.3"
   gem.add_dependency "pg", "~> 0.17.0"
   gem.add_development_dependency "rake"

data/lib/fluent/plugin/out_redshift.rb

@@ -4,6 +4,8 @@ module Fluent
 class RedshiftOutput < BufferedOutput
   Fluent::Plugin.register_output('redshift', self)
 
+  NULL_CHAR_FOR_COPY = "\\N"
+
   # ignore load table error. (invalid data format)
   IGNORE_REDSHIFT_ERROR_REGEXP = /^ERROR:  Load into table '[^']+' failed\./
 
@@ -20,8 +22,8 @@ class RedshiftOutput < BufferedOutput
 
   config_param :record_log_tag, :string, :default => 'log'
   # s3
-  config_param :aws_key_id, :string
-  config_param :aws_sec_key, :string
+  config_param :aws_key_id, :string, :secret => true
+  config_param :aws_sec_key, :string, :secret => true
   config_param :s3_bucket, :string
   config_param :s3_endpoint, :string, :default => nil
   config_param :path, :string, :default => ""
@@ -32,7 +34,7 @@ class RedshiftOutput < BufferedOutput
   config_param :redshift_port, :integer, :default => 5439
   config_param :redshift_dbname, :string
   config_param :redshift_user, :string
-  config_param :redshift_password, :string
+  config_param :redshift_password, :string, :secret => true
   config_param :redshift_tablename, :string
   config_param :redshift_schemaname, :string, :default => nil
   config_param :redshift_copy_base_options, :string , :default => "FILLRECORD ACCEPTANYDATE TRUNCATECOLUMNS"
@@ -41,6 +43,8 @@ class RedshiftOutput < BufferedOutput
   # file format
   config_param :file_type, :string, :default => nil  # json, tsv, csv, msgpack
   config_param :delimiter, :string, :default => nil
+  # maintenance
+  config_param :maintenance_file_path, :string, :default => nil
   # for debug
   config_param :log_suffix, :string, :default => ''
 
@@ -61,6 +65,7 @@ class RedshiftOutput < BufferedOutput
     $log.debug format_log("redshift file_type:#{@file_type} delimiter:'#{@delimiter}'")
     @table_name_with_schema = [@redshift_schemaname, @redshift_tablename].compact.join('.')
     @copy_sql_template = "copy #{@table_name_with_schema} from '%s' CREDENTIALS 'aws_access_key_id=#{@aws_key_id};aws_secret_access_key=%s' delimiter '#{@delimiter}' GZIP ESCAPE #{@redshift_copy_base_options} #{@redshift_copy_options};"
+    @maintenance_monitor = MaintenanceMonitor.new(@maintenance_file_path)
   end
 
   def start
@@ -88,6 +93,7 @@ class RedshiftOutput < BufferedOutput
 
   def write(chunk)
     $log.debug format_log("start creating gz.")
+    @maintenance_monitor.check_maintenance!
 
     # create a gz file
     tmp = Tempfile.new("s3-")
@@ -180,12 +186,7 @@ class RedshiftOutput < BufferedOutput
           tsv_text = hash_to_table_text(redshift_table_columns, hash, delimiter)
           gzw.write(tsv_text) if tsv_text and not tsv_text.empty?
         rescue => e
-          if json?
-            $log.error format_log("failed to create table text from json. text=(#{record[@record_log_tag]})"), :error=>e.to_s
-          else
-            $log.error format_log("failed to create table text from msgpack. text=(#{record[@record_log_tag]})"), :error=>e.to_s
-          end
-
+          $log.error format_log("failed to create table text from #{@file_type}. text=(#{record[@record_log_tag]})"), :error=>e.to_s
           $log.error_backtrace
         end
       end
@@ -220,18 +221,9 @@ class RedshiftOutput < BufferedOutput
     return "" unless hash
 
     # extract values from hash
-    val_list = redshift_table_columns.collect do |cn|
-      val = hash[cn]
-      val = JSON.generate(val) if val.kind_of?(Hash) or val.kind_of?(Array)
-
-      if val.to_s.empty?
-        nil
-      else
-        val.to_s
-      end
-    end
+    val_list = redshift_table_columns.collect {|cn| hash[cn]}
 
-    if val_list.all?{|v| v.nil? or v.empty?}
+    if val_list.all?{|v| v.nil?}
       $log.warn format_log("no data match for table columns on redshift. data=#{hash} table_columns=#{redshift_table_columns}")
       return ""
     end
@@ -240,14 +232,22 @@ class RedshiftOutput < BufferedOutput
   end
 
   def generate_line_with_delimiter(val_list, delimiter)
-    val_list = val_list.collect do |val|
-      if val.nil? or val.empty?
-        ""
+    val_list.collect do |val|
+      case val
+      when nil
+        NULL_CHAR_FOR_COPY
+      when ''
+        ''
+      when Hash, Array
+        escape_text_for_copy(JSON.generate(val))
       else
-        val.gsub(/\\/, "\\\\\\").gsub(/\t/, "\\\t").gsub(/\n/, "\\\n") # escape tab, newline and backslash
+        escape_text_for_copy(val.to_s)
       end
-    end
-    val_list.join(delimiter) + "\n"
+    end.join(delimiter) + "\n"
+  end
+
+  def escape_text_for_copy(val)
+    val.gsub(/\\|\t|\n/, {"\\" => "\\\\", "\t" => "\\\t", "\n" => "\\\n"})  # escape tab, newline and backslash
   end
 
   def create_s3path(bucket, path)
@@ -361,8 +361,26 @@ class RedshiftOutput < BufferedOutput
       sql
     end
   end
-end
 
+  class MaintenanceError < StandardError
+  end
+
+  class MaintenanceMonitor
+    def initialize(maintenance_file_path)
+      @file_path = maintenance_file_path
+    end
+
+    def in_maintenance?
+      !!(@file_path && File.exists?(@file_path))
+    end
+
+    def check_maintenance!
+      if in_maintenance?
+        raise MaintenanceError.new("Service is in maintenance mode - maintenance_file_path:#{@file_path}")
+      end
+    end
+  end
+end
 
 
 end

data/test/plugin/test_out_redshift.rb

@@ -15,6 +15,8 @@ class RedshiftOutputTest < Test::Unit::TestCase
     PG::Error.module_eval { attr_accessor :result}
   end
 
+  MAINTENANCE_FILE_PATH_FOR_TEST = "/tmp/fluentd_redshift_plugin_test_maintenance"
+
   CONFIG_BASE= %[
     aws_key_id test_key_id
     aws_sec_key test_sec_key
@@ -28,6 +30,7 @@ class RedshiftOutputTest < Test::Unit::TestCase
     buffer_type memory
     utc
     log_suffix id:5 host:localhost
+    maintenance_file_path #{MAINTENANCE_FILE_PATH_FOR_TEST}
   ]
   CONFIG_CSV= %[
     #{CONFIG_BASE}
@@ -107,6 +110,7 @@ class RedshiftOutputTest < Test::Unit::TestCase
     assert_equal "csv", d.instance.file_type
     assert_equal ",", d.instance.delimiter
     assert_equal true, d.instance.utc
+    assert_equal MAINTENANCE_FILE_PATH_FOR_TEST, d.instance.maintenance_file_path
   end
   def test_configure_with_schemaname
     d = create_driver(CONFIG_JSON_WITH_SCHEMA)
@@ -311,7 +315,7 @@ class RedshiftOutputTest < Test::Unit::TestCase
   end
 
   def test_write_with_json
-    setup_mocks(%[val_a\tval_b\t\t\t\t\t\t\n\t\tval_c\tval_d\t\t\t\t\n])
+    setup_mocks(%[val_a\tval_b\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\n\\N\t\\N\tval_c\tval_d\t\\N\t\\N\t\\N\t\\N\n])
     setup_tempfile_mock_to_be_closed
     d_json = create_driver(CONFIG_JSON)
     emit_json(d_json)
@@ -319,7 +323,7 @@ class RedshiftOutputTest < Test::Unit::TestCase
   end
 
   def test_write_with_json_hash_value
-    setup_mocks("val_a\t{\"foo\":\"var\"}\t\t\t\t\t\t\n\t\tval_c\tval_d\t\t\t\t\n")
+    setup_mocks("val_a\t{\"foo\":\"var\"}\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\n\\N\t\\N\tval_c\tval_d\t\\N\t\\N\t\\N\t\\N\n")
     d_json = create_driver(CONFIG_JSON)
     d_json.emit({"log" => %[{"key_a" : "val_a", "key_b" : {"foo" : "var"}}]} , DEFAULT_TIME)
     d_json.emit(RECORD_JSON_B, DEFAULT_TIME)
@@ -327,7 +331,7 @@ class RedshiftOutputTest < Test::Unit::TestCase
   end
 
   def test_write_with_json_array_value
-    setup_mocks("val_a\t[\"foo\",\"var\"]\t\t\t\t\t\t\n\t\tval_c\tval_d\t\t\t\t\n")
+    setup_mocks("val_a\t[\"foo\",\"var\"]\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\n\\N\t\\N\tval_c\tval_d\t\\N\t\\N\t\\N\t\\N\n")
     d_json = create_driver(CONFIG_JSON)
     d_json.emit({"log" => %[{"key_a" : "val_a", "key_b" : ["foo", "var"]}]} , DEFAULT_TIME)
     d_json.emit(RECORD_JSON_B, DEFAULT_TIME)
@@ -335,13 +339,20 @@ class RedshiftOutputTest < Test::Unit::TestCase
   end
 
   def test_write_with_json_including_tab_newline_quote
-    setup_mocks("val_a_with_\\\t_tab_\\\n_newline\tval_b_with_\\\\_quote\t\t\t\t\t\t\n\t\tval_c\tval_d\t\t\t\t\n")
+    setup_mocks("val_a_with_\\\t_tab_\\\n_newline\tval_b_with_\\\\_quote\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\n\\N\t\\N\tval_c\tval_d\t\\N\t\\N\t\\N\t\\N\n")
     d_json = create_driver(CONFIG_JSON)
     d_json.emit({"log" => %[{"key_a" : "val_a_with_\\t_tab_\\n_newline", "key_b" : "val_b_with_\\\\_quote"}]} , DEFAULT_TIME)
     d_json.emit(RECORD_JSON_B, DEFAULT_TIME)
     assert_equal true, d_json.run
   end
 
+  def test_write_with_json_empty_text_value
+    setup_mocks(%[val_a\t\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\n])
+    d_json = create_driver(CONFIG_JSON)
+    d_json.emit({"log" => %[{"key_a" : "val_a", "key_b" : ""}]} , DEFAULT_TIME)
+    assert_equal true, d_json.run
+  end
+
   def test_write_with_json_no_data
     setup_mocks("")
     d_json = create_driver(CONFIG_JSON)
@@ -351,7 +362,7 @@ class RedshiftOutputTest < Test::Unit::TestCase
   end
 
   def test_write_with_json_invalid_one_line
-    setup_mocks(%[\t\tval_c\tval_d\t\t\t\t\n])
+    setup_mocks(%[\\N\t\\N\tval_c\tval_d\t\\N\t\\N\t\\N\t\\N\n])
     d_json = create_driver(CONFIG_JSON)
     d_json.emit({"log" => %[}}]}, DEFAULT_TIME)
     d_json.emit(RECORD_JSON_B, DEFAULT_TIME)
@@ -359,7 +370,7 @@ class RedshiftOutputTest < Test::Unit::TestCase
   end
 
   def test_write_with_json_no_available_data
-    setup_mocks(%[val_a\tval_b\t\t\t\t\t\t\n])
+    setup_mocks(%[val_a\tval_b\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\n])
     d_json = create_driver(CONFIG_JSON)
     d_json.emit(RECORD_JSON_A, DEFAULT_TIME)
     d_json.emit({"log" => %[{"key_o" : "val_o", "key_p" : "val_p"}]}, DEFAULT_TIME)
@@ -367,14 +378,14 @@ class RedshiftOutputTest < Test::Unit::TestCase
   end
 
   def test_write_with_msgpack
-    setup_mocks(%[val_a\tval_b\t\t\t\t\t\t\n\t\tval_c\tval_d\t\t\t\t\n])
+    setup_mocks(%[val_a\tval_b\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\n\\N\t\\N\tval_c\tval_d\t\\N\t\\N\t\\N\t\\N\n])
     d_msgpack = create_driver(CONFIG_MSGPACK)
     emit_msgpack(d_msgpack)
     assert_equal true, d_msgpack.run
   end
 
   def test_write_with_msgpack_hash_value
-    setup_mocks("val_a\t{\"foo\":\"var\"}\t\t\t\t\t\t\n\t\tval_c\tval_d\t\t\t\t\n")
+    setup_mocks("val_a\t{\"foo\":\"var\"}\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\n\\N\t\\N\tval_c\tval_d\t\\N\t\\N\t\\N\t\\N\n")
     d_msgpack = create_driver(CONFIG_MSGPACK)
     d_msgpack.emit({"key_a" => "val_a", "key_b" => {"foo" => "var"}} , DEFAULT_TIME)
     d_msgpack.emit(RECORD_MSGPACK_B, DEFAULT_TIME)
@@ -382,7 +393,7 @@ class RedshiftOutputTest < Test::Unit::TestCase
   end
 
   def test_write_with_msgpack_array_value
-    setup_mocks("val_a\t[\"foo\",\"var\"]\t\t\t\t\t\t\n\t\tval_c\tval_d\t\t\t\t\n")
+    setup_mocks("val_a\t[\"foo\",\"var\"]\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\n\\N\t\\N\tval_c\tval_d\t\\N\t\\N\t\\N\t\\N\n")
     d_msgpack = create_driver(CONFIG_MSGPACK)
     d_msgpack.emit({"key_a" => "val_a", "key_b" => ["foo", "var"]} , DEFAULT_TIME)
     d_msgpack.emit(RECORD_MSGPACK_B, DEFAULT_TIME)
@@ -390,7 +401,7 @@ class RedshiftOutputTest < Test::Unit::TestCase
   end
 
   def test_write_with_msgpack_including_tab_newline_quote
-    setup_mocks("val_a_with_\\\t_tab_\\\n_newline\tval_b_with_\\\\_quote\t\t\t\t\t\t\n\t\tval_c\tval_d\t\t\t\t\n")
+    setup_mocks("val_a_with_\\\t_tab_\\\n_newline\tval_b_with_\\\\_quote\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\n\\N\t\\N\tval_c\tval_d\t\\N\t\\N\t\\N\t\\N\n")
     d_msgpack = create_driver(CONFIG_MSGPACK)
     d_msgpack.emit({"key_a" => "val_a_with_\t_tab_\n_newline", "key_b" => "val_b_with_\\_quote"} , DEFAULT_TIME)
     d_msgpack.emit(RECORD_MSGPACK_B, DEFAULT_TIME)
@@ -406,7 +417,7 @@ class RedshiftOutputTest < Test::Unit::TestCase
   end
 
   def test_write_with_msgpack_no_available_data
-    setup_mocks(%[val_a\tval_b\t\t\t\t\t\t\n])
+    setup_mocks(%[val_a\tval_b\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\n])
     d_msgpack = create_driver(CONFIG_MSGPACK)
     d_msgpack.emit(RECORD_MSGPACK_A, DEFAULT_TIME)
     d_msgpack.emit({"key_o" => "val_o", "key_p" => "val_p"}, DEFAULT_TIME)
@@ -466,10 +477,21 @@ class RedshiftOutputTest < Test::Unit::TestCase
   end
 
   def test_write_with_json_fetch_column_with_schema
-    setup_mocks(%[val_a\tval_b\t\t\t\t\t\t\n\t\tval_c\tval_d\t\t\t\t\n],
+    setup_mocks(%[val_a\tval_b\t\\N\t\\N\t\\N\t\\N\t\\N\t\\N\n\\N\t\\N\tval_c\tval_d\t\\N\t\\N\t\\N\t\\N\n],
                schema_name: 'test_schema')
     d_json = create_driver(CONFIG_JSON_WITH_SCHEMA)
     emit_json(d_json)
     assert_equal true, d_json.run
   end
+
+  def test_maintenance_mode
+    flexmock(File).should_receive(:exists?).with(MAINTENANCE_FILE_PATH_FOR_TEST).and_return(true)
+
+    d_json = create_driver(CONFIG_JSON)
+    emit_json(d_json)
+    assert_raise(Fluent::RedshiftOutput::MaintenanceError,
+                 "Service is in maintenance mode - maintenance_file_path:#{MAINTENANCE_FILE_PATH_FOR_TEST}") {
+      d_json.run
+    }
+  end
 end

metadata

@@ -1,29 +1,35 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-redshift
 version: !ruby/object:Gem::Version
-  version: 0.0.6
+  version: 0.1.0
 platform: ruby
 authors:
 - Masashi Miyazaki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-08-05 00:00:00.000000000 Z
+date: 2015-08-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.10.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: '2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - '>='
       - !ruby/object:Gem::Version
         version: 0.10.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: '2'
 - !ruby/object:Gem::Dependency
   name: aws-sdk-v1
   requirement: !ruby/object:Gem::Requirement