fluent-plugin-redshift 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6ff3e820b9cccac040efc73a11cf261f4a307dda
-  data.tar.gz: f00f6cc70b5cf75ac399ba65cc25f773d02123db
+  metadata.gz: 2b88c3334997e74901efda542d441ee184e6fba6
+  data.tar.gz: ddaf48cb0097b54d1232057d17f6e2b80f1d6675
 SHA512:
-  metadata.gz: 096462829885b50f9cd01852843c80fd8096d028d8e9c0f45160fc2be4f3f66e43f7e28cf4b137c79f46bbbaad8f042693ee72e9b00f793e16a2dd280d1e7b9b
-  data.tar.gz: 739976ff4077c4ce34c51ee311dcdc10dd186f9f45ab39414292f8f49b9d87d2ad2dec5f2540f3f465bf6604932e05c115d2da5250b619236df580f7840ebf29
+  metadata.gz: 5e72f127433af970ac505de35235e0879eb9390f1bf3b43e20fea896d54ed25347071efd6b42e769bf7e77525e49355f7b085c43f9fbad5c464afd0182c02a5a
+  data.tar.gz: 082a35783e38b79f31a216ef01a3625e6c492d007f58b186a0edad1a4dcae8c1fba72a9c5bfa850f9089937aab86974cafb797ccf34b9e2acfa3733d7a6de093

data/VERSION

@@ -1 +1 @@
-0.1.1
+0.1.2

data/lib/fluent/plugin/out_redshift.rb

@@ -64,7 +64,7 @@ DESC
 Schema name to store data. By default, this option is not
 Set and find table without schema as your own search_path.
 DESC
-  config_param :redshift_copy_base_options, :string , :default => "FILLRECORD ACCEPTANYDATE TRUNCATECOLUMNS"
+  config_param :redshift_copy_base_options, :string , :default => "ESCAPE FILLRECORD ACCEPTANYDATE TRUNCATECOLUMNS"
   config_param :redshift_copy_options, :string , :default => nil
   config_param :redshift_connect_timeout, :integer, :default => 10,
                :desc => "Maximum time to wait for connection to succeed."
@@ -214,7 +214,10 @@ DESC
                   else
                     "CREDENTIALS 'aws_iam_role=#{@aws_iam_role}'"
                   end
-   "copy #{@table_name_with_schema}#{copy_columns} from '%s' #{credentials} delimiter '#{@delimiter}' GZIP ESCAPE #{@redshift_copy_base_options} #{@redshift_copy_options};"
+    escape = if !@redshift_copy_base_options.include?('ESCAPE') && (json? || msgpack?)
+               " ESCAPE"
+             end
+   "copy #{@table_name_with_schema}#{copy_columns} from '%s' #{credentials} delimiter '#{@delimiter}' GZIP#{escape} #{@redshift_copy_base_options} #{@redshift_copy_options};"
   end
 
 

data/test/plugin/test_out_redshift.rb

@@ -107,13 +107,14 @@ class RedshiftOutputTest < Test::Unit::TestCase
     assert_equal "test_password", d.instance.redshift_password
     assert_equal "test_table", d.instance.redshift_tablename
     assert_equal nil, d.instance.redshift_schemaname
-    assert_equal "FILLRECORD ACCEPTANYDATE TRUNCATECOLUMNS", d.instance.redshift_copy_base_options
+    assert_equal "ESCAPE FILLRECORD ACCEPTANYDATE TRUNCATECOLUMNS", d.instance.redshift_copy_base_options
     assert_equal nil, d.instance.redshift_copy_options
     assert_equal "csv", d.instance.file_type
     assert_equal ",", d.instance.delimiter
     assert_equal true, d.instance.utc
     assert_equal MAINTENANCE_FILE_PATH_FOR_TEST, d.instance.maintenance_file_path
     assert_equal nil, d.instance.redshift_copy_columns
+    assert_equal true, d.instance.instance_variable_get("@copy_sql_template").include?('ESCAPE')
   end
   def test_configure_with_schemaname
     d = create_driver(CONFIG_JSON_WITH_SCHEMA)
@@ -147,16 +148,28 @@ class RedshiftOutputTest < Test::Unit::TestCase
     d1 = create_driver(CONFIG_TSV)
     assert_equal "tsv", d1.instance.file_type
     assert_equal "\t", d1.instance.delimiter
+    assert_equal true, d1.instance.instance_variable_get("@copy_sql_template").include?('ESCAPE')
+  end
+  def test_configure_tsv_without_copy_escape
+    d1 = create_driver(CONFIG_TSV + "\n  redshift_copy_base_options FILLRECORD ACCEPTANYDATE TRUNCATECOLUMNS")
+    assert_equal false, d1.instance.instance_variable_get("@copy_sql_template").include?('ESCAPE')
   end
   def test_configure_json
     d2 = create_driver(CONFIG_JSON)
     assert_equal "json", d2.instance.file_type
     assert_equal "\t", d2.instance.delimiter
+    assert_match /^copy test_table.*ESCAPE.*/, d2.instance.instance_variable_get("@copy_sql_template")
+  end
+  def test_configure_json_without_copy_escape
+    d1 = create_driver(CONFIG_JSON + "\n  redshift_copy_base_options FILLRECORD ACCEPTANYDATE TRUNCATECOLUMNS")
+    # It should add ESCAPE option since json value is escaped anyway by our code
+    assert_equal true, d1.instance.instance_variable_get("@copy_sql_template").include?('ESCAPE')
   end
   def test_configure_msgpack
     d2 = create_driver(CONFIG_MSGPACK)
     assert_equal "msgpack", d2.instance.file_type
     assert_equal "\t", d2.instance.delimiter
+    assert_match /^copy test_table.*ESCAPE.*/, d2.instance.instance_variable_get("@copy_sql_template")
   end
   def test_configure_original_file_type
     d3 = create_driver(CONFIG_PIPE_DELIMITER)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-redshift
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Masashi Miyazaki
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-07-06 00:00:00.000000000 Z
+date: 2016-07-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd