fluent-plugin-mysql-select-insert 0.1.0 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a43d623e098edb37e7977ad86d9dfdfdc7748c129f0dc6127f773d755d19f8e0
-  data.tar.gz: 89c6745c98fc33f0961b476cfa668129a4a07c4bf6e5fe2d21540e64950774fb
+  metadata.gz: 0f67d87a1bfad110f3c66632eacc9768b2203457209447ff081c07561c5e6356
+  data.tar.gz: 3634f61c95855e08e2e1742b6f1b154a916c15825ec3ea45eb4da1de324a7e09
 SHA512:
-  metadata.gz: d0dab5f92c67d336c6ec60505d4755e21ec941775a2cac4d8cbfbd7aeb3e068652ad9bbdf52393ebee3b5a106758b01a5eeccc62ecb189155b4ea609f9360e92
-  data.tar.gz: 995963147d81ef09a96111e9b112d9177b127ebd3459a97b71c187ddd71392670db9974e6cb2126afafadfeb4755a14339811da8ff125bfaba323b4f66321260
+  metadata.gz: 66341a37c23a9ce9f05948d30b17ef90ebfcfc44f30a23d13d177b3e859b34a82f8af632766986995f45f2a6f8397c64277038e68ded1b8f903ca4832b9a9237
+  data.tar.gz: 2e5695b27761643f506b048e485468c5956a46b6dbd51b258e5b72fd20c1f20ad1b41b070ae314370c64746c5ce43f505f26c2b9dfbe9e7fcc69685f0696508d

data/.travis.yml

@@ -1,8 +1,10 @@
-sudo: false
 language: ruby
 cache: bundler
 rvm:
-  - 2.5.0
+  - 2.5.6
+  - 2.6.4
+
+services:
+  - mysql
 before_install:
-  - gem install bundler -v 1.16.1
   - mysql -e 'CREATE DATABASE IF NOT EXISTS fluent_plugin_mysql_select_insert;'

data/LICENSE

@@ -187,7 +187,7 @@
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-   Copyright [yyyy] [name of copyright owner]
+   Copyright 2018-2019 Takeshi Arabiki (abicky)
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

data/README.md

@@ -39,6 +39,7 @@ select_query | SELECT query without WHERE clause to insert records | (required)
 condition_column | Column used in WHERE clause to compare with values of 'condition_key' in events | (required)
 condition_key | Key whose value is used in WHERE clause to compare with 'condition_column' | (required)
 extra_condition | Extra condition used in WHERE clause. You can specify placeholders like 'col1 = ? AND col2 = ?, ${tag[0]}, ${tag[1]}'. The metadata is extracted in the second or following arguments. | (empty array)
+inserted_columns | Columns to be inserted. You should insert all columns by the select query if you don't specify the value. | nil
 ignore    | Add 'IGNORE' modifier to INSERT statement | false
 
 ## Example Configuration

data/fluent-plugin-mysql-select-insert.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-mysql-select-insert"
-  spec.version       = "0.1.0"
+  spec.version       = "0.1.1"
   spec.authors       = ["abicky"]
   spec.email         = ["takeshi.arabiki@gmail.com"]
 
@@ -20,9 +20,9 @@ Gem::Specification.new do |spec|
   spec.test_files    = test_files
   spec.require_paths = ["lib"]
 
-  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "bundler", ">= 1.16", "< 3"
   spec.add_development_dependency "rake", "~> 12.0"
   spec.add_development_dependency "test-unit", "~> 3.0"
-  spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
+  spec.add_runtime_dependency "fluentd", [">= 1.2.0", "< 2"]
   spec.add_runtime_dependency "mysql2-cs-bind"
 end

data/lib/fluent/plugin/out_mysql_select_insert.rb

@@ -42,6 +42,8 @@ module Fluent
         desc: "Key whose value is used in WHERE clause to compare with 'condition_column'"
       config_param :extra_condition, :array, value_type: :string, default: [],
         desc: "Extra condition used in WHERE clause. You can specify placeholders like 'col1 = ? AND col2 = ?, ${tag[0]}, ${tag[1]}'. The metadata is extracted in the second or following arguments."
+      config_param :inserted_columns, :array, value_type: :string, default: nil,
+        desc: "Columns to be inserted. You should insert all columns by the select query if you don't specify the value."
 
       config_param :ignore, :bool, default: false,
         desc: "Add 'IGNORE' modifier to INSERT statement"
@@ -56,6 +58,9 @@ module Fluent
         if select_query =~ /\bwhere\b/i
           fail Fluent::ConfigError, "You can't specify WHERE clause in 'select_query'"
         end
+        if select_query !~ /\A\s*select\b/i
+          fail Fluent::ConfigError, "'select_query' should begin with 'SELECT'"
+        end
       end
 
       def write(chunk)
@@ -64,19 +69,29 @@ module Fluent
         chunk.msgpack_each do |time, record|
           condition_values << "'#{client.escape(record[condition_key])}'"
         end
+
         sql = <<~SQL
-          INSERT #{"IGNORE" if ignore} INTO `#{table}` #{select_query}
+          INSERT #{"IGNORE" if ignore} INTO `#{table}` #{"(#{inserted_columns.join(",")})" if inserted_columns}
+          #{select_query}
           WHERE #{condition_column} IN (#{condition_values.join(",")})
         SQL
-
         sql << " AND (#{extra_condition[0]})" unless extra_condition.empty?
-        if extra_condition.size < 2
-          client.query(sql)
-        else
-          require "mysql2-cs-bind"
-          bound_params = extra_condition[1..-1].map { |c| extract_placeholders(c, chunk.metadata) }
-          client.xquery(sql, bound_params)
+
+        bound_params = extra_condition[1..-1]&.map { |c| extract_placeholders(c, chunk.metadata) }
+        begin
+          if bound_params.nil? || bound_params.empty?
+            client.query(sql)
+          else
+            require "mysql2-cs-bind"
+            client.xquery(sql, bound_params)
+          end
+        rescue Mysql2::Error => e
+          if e.message.start_with?("Column count doesn't match value count")
+            raise Fluent::UnrecoverableError, "#{e.class}: #{e}"
+          end
+          raise
         end
+
         client.close
       end
 

data/test/plugin/test_out_mysql_select_insert.rb

@@ -4,66 +4,81 @@ require "fluent/plugin/out_mysql_select_insert"
 class MysqlSelectInsertOutputTest < Test::Unit::TestCase
   class << self
     def startup
-      new_client.query(<<~SQL)
+      client = new_client
+      client.query(<<~SQL)
         CREATE TABLE IF NOT EXISTS `devices` (
           `id` int(10) unsigned NOT NULL,
           `app_id` int(10) unsigned NOT NULL,
           `uuid` char(36) NOT NULL,
           PRIMARY KEY (`id`),
           UNIQUE KEY (`app_id`, `uuid`)
-        );
+        )
+      SQL
+      client.query(<<~SQL)
         CREATE TABLE IF NOT EXISTS `users` (
           `id` int(10) unsigned NOT NULL,
           `device_id` int(10) unsigned NOT NULL,
           PRIMARY KEY (`id`)
-        );
+        )
+      SQL
+      client.query(<<~SQL)
         CREATE TABLE IF NOT EXISTS `accessed_users` (
           `user_id` int(10) unsigned NOT NULL,
           PRIMARY KEY (`user_id`)
-        );
+        )
       SQL
     end
 
     def shutdown
-      new_client.query(<<~SQL)
-        DROP TABLE IF EXISTS `devices`;
-        DROP TABLE IF EXISTS `users`;
-        DROP TABLE IF EXISTS `accessed_users`;
-      SQL
+      client = new_client
+      client.query("DROP TABLE IF EXISTS `devices`")
+      client.query("DROP TABLE IF EXISTS `users`")
+      client.query("DROP TABLE IF EXISTS `accessed_users`")
     end
 
     def new_client
       Mysql2::Client.new(
         username: "root",
         database: "fluent_plugin_mysql_select_insert",
-        flags: Mysql2::Client::MULTI_STATEMENTS,
       )
     end
   end
 
+  def suppress_errors
+    prev_value = Thread.report_on_exception
+    # Supporess errors from flush treads
+    Thread.report_on_exception = false
+    # Suppress errors like "unexpected error while after_shutdown"
+    capture_stdout { yield }
+  ensure
+    Thread.report_on_exception = prev_value
+  end
+
   setup do
     Fluent::Test.setup
 
-    self.class.new_client.query(<<~SQL)
+    client = self.class.new_client
+    client.query(<<~SQL)
       INSERT INTO `devices` (`id`, `app_id`, `uuid`) VALUES
         (1, 1, '03449258-29ce-403c-900a-a2c6ea1d09a2'),
         (2, 1, '11aebc82-b661-44ab-951d-d1618058699a'),
         (3, 2, '2c9a7bb7-aec2-441c-ade0-edf31fdacb43'),
-        (4, 2, '3eae0b4b-bdd1-4c82-b04a-5335535f5b7b');
+        (4, 2, '3eae0b4b-bdd1-4c82-b04a-5335535f5b7b')
+    SQL
+    client.query(<<~SQL)
       INSERT INTO `users` (`id`, `device_id`) VALUES
         (1001, 1),
         (1002, 2),
         (1003, 3),
-        (1004, 4);
+        (1004, 4)
     SQL
   end
 
   teardown do
-    self.class.new_client.query(<<~SQL)
-      TRUNCATE TABLE `devices`;
-      TRUNCATE TABLE `users`;
-      TRUNCATE TABLE `accessed_users`;
-    SQL
+    client = self.class.new_client
+    client.query("TRUNCATE TABLE `devices`")
+    client.query("TRUNCATE TABLE `users`")
+    client.query("TRUNCATE TABLE `accessed_users`")
   end
 
   CONFIG = <<~CONF
@@ -196,22 +211,22 @@ class MysqlSelectInsertOutputTest < Test::Unit::TestCase
       condition_column devices.uuid
     CONF
 
-    test "write with ignore false" do
-      # XXX: "Mysql2::Error: Table 'fluent_plugin_mysql_select_insert.accessed_users' doesn't exist" occurs without sleep
-      sleep 1
+    setup do
       self.class.new_client.query("INSERT INTO `accessed_users` VALUES (1001)")
+    end
 
+    test "write with ignore false" do
       d = create_driver("#{base_config}\n ignore false")
       assert_raise Mysql2::Error do
-        d.run(default_tag: "tag") do
-          d.feed(event_time, { "uuid" => "03449258-29ce-403c-900a-a2c6ea1d09a2", "app_id" => 1 })
+        suppress_errors do
+          d.run(default_tag: "tag") do
+            d.feed(event_time, { "uuid" => "03449258-29ce-403c-900a-a2c6ea1d09a2", "app_id" => 1 })
+          end
         end
       end
     end
 
     test "write with ignore true" do
-      self.class.new_client.query("INSERT INTO `accessed_users` VALUES (1001)")
-
       d = create_driver("#{base_config}\n ignore true")
       assert_nothing_raised do
         d.run(default_tag: "tag") do
@@ -221,6 +236,36 @@ class MysqlSelectInsertOutputTest < Test::Unit::TestCase
     end
   end
 
+  sub_test_case "'inserted_columns' is specified" do
+    setup do
+      self.class.new_client.query("ALTER TABLE `accessed_users` ADD `extra_column` VARCHAR(63) DEFAULT 'default'")
+    end
+
+    teardown do
+      self.class.new_client.query("ALTER TABLE `accessed_users` DROP `extra_column`")
+    end
+
+    test "write with inserted_columns" do
+      d = create_driver("#{CONFIG}\n inserted_columns 'user_id'")
+      d.run(default_tag: "tag") do
+        d.feed(event_time, { "uuid" => "03449258-29ce-403c-900a-a2c6ea1d09a2", "app_id" => 1 })
+      end
+      result = self.class.new_client.query('SELECT * FROM `accessed_users`')
+      assert_equal [{ "user_id" => 1001, "extra_column" => "default" }], result.to_a
+    end
+
+    test "write with invalid inserted_columns" do
+      d = create_driver("#{CONFIG}\n inserted_columns 'user_id, extra_column'")
+
+      d.run(default_tag: "tag") do
+        d.feed(event_time, { "uuid" => "03449258-29ce-403c-900a-a2c6ea1d09a2", "app_id" => 1 })
+      end
+      assert do
+        d.logs.any? { |l| l.include?("got unrecoverable error") && l.include?("Column count doesn't match") }
+      end
+    end
+  end
+
   sub_test_case "WHERE clause is specified in 'select_query'" do
     test "configure" do
       assert_raise Fluent::ConfigError do
@@ -248,6 +293,33 @@ class MysqlSelectInsertOutputTest < Test::Unit::TestCase
     end
   end
 
+  sub_test_case "'select_query' doesn't begin with 'SELECT'" do
+    test "configure" do
+      assert_raise Fluent::ConfigError do
+        create_driver(<<~CONF)
+          database fluent_plugin_mysql_select_insert
+          username root
+
+          table accessed_users
+
+          select_query "(id) SELECT
+              users.id
+            FROM
+              users
+              INNER JOIN
+                devices
+              ON
+                devices.id = users.device_id
+            WHERE
+              app_id = 1"
+
+          condition_key uuid
+          condition_column devices.uuid
+        CONF
+      end
+    end
+  end
+
   private
 
   def create_driver(conf = CONFIG)

metadata

@@ -1,29 +1,35 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-mysql-select-insert
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - abicky
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-04-27 00:00:00.000000000 Z
+date: 2019-09-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.16'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.16'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -58,7 +64,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.10
+        version: 1.2.0
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -68,7 +74,7 @@ dependencies:
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.14.10
+        version: 1.2.0
     - - "<"
       - !ruby/object:Gem::Version
         version: '2'
@@ -121,8 +127,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.3
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: Fluentd output plugin to insert records by SELECT query.