logstash-output-analyticdb 5.4.0.6 → 5.4.0.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 05f92d45f98c4f9dd2801ffdd1e56bb5f4fe3550a383833773505249997a4c5d
-  data.tar.gz: 52cd4f1ad8addbc6b5a970b859755e9a4925b102c3411f156bb986d9763561fb
+  metadata.gz: 27a21842e6bfba6088e5e56ece9763a7ef3288eb135ccf0689bd632d2784c676
+  data.tar.gz: 772b83405d4981e98bb8685490591ee100a6a4f9c02a55b98e0635c365f4dc26
 SHA512:
-  metadata.gz: cd0bfc9d2a2ff6d9066bf3925012c1b8e7d835f69b4233f2ddc28b178d6e942534c0a67d328da76b0b35d73ca4bb28445f38dec422aaa3f5c1d6bca0b8ab3c3f
-  data.tar.gz: c711267abf70c0bec1adf2263b197a1fb603792d982db07984c0a61717c51a5b2aaa1b53afa0b5ed80b0d98702ec2a7c39d188d4c21e18d49291847f428cfacd
+  metadata.gz: b41bb657102f6a2512ac3f503187f55f42219db01ee9fd0f3f2d7e6bfe93d35d70aff2a1634e7908d58d8a72256ab8994e2074b1ce5a3609228ba5a0a9fbab25
+  data.tar.gz: 725a27df6db012727913bced51dba68a7c2fdb642483f637a5fdbc85be19014b7a0d825b22ef5cbb5c7bf24e04ab5d4b38e485fcfa1e7e3b566576318e10029d

data/README.md

@@ -6,14 +6,6 @@ This plugin is provided as an external plugin and is not part of the Logstash pr
 This plugin allows you to output to AnalyticDB database, using JDBC adapters.
 See below for tested adapters, and example configurations.
 
-## Support & release schedule
-I no longer have time at work to maintain this plugin in step with Logstash's releases, and I am not completely immersed in the Logstash ecosystem. If something is broken for you I will do my best to help, but I cannot guarantee timeframes.
-
-Pull requests are always welcome. 
-
-## Changelog
-See CHANGELOG.md
-
 ## Versions
 Released versions are available via rubygems, and typically tagged.
 

data/lib/logstash/outputs/analyticdb.rb

@@ -94,7 +94,7 @@ class LogStash::Outputs::Analyticdb < LogStash::Outputs::Base
   # Maximum number of sequential failed attempts, before we stop retrying.
   # If set to < 1, then it will infinitely retry.
   # At the default values this is a little over 10 minutes
-  config :max_flush_exceptions, validate: :number, default: 10
+  config :max_flush_exceptions, validate: :number, default: 100
 
   config :max_repeat_exceptions, obsolete: 'This has been replaced by max_flush_exceptions - which behaves slightly differently. Please check the documentation.'
   config :max_repeat_exceptions_time, obsolete: 'This is no longer required'
@@ -108,6 +108,8 @@ class LogStash::Outputs::Analyticdb < LogStash::Outputs::Base
 
   config :commit_size, validate: :number, default: 32768
 
+  config :skip_exception, validate: :boolean, default: false
+
   def register
     @logger.info('JDBC - Starting up')
 
@@ -215,8 +217,6 @@ class LogStash::Outputs::Analyticdb < LogStash::Outputs::Base
     connection = nil
     statement = nil
     events_to_retry = []
-    insert_sql = ""
-    sql_len = 0
     is_insert_err = false
 
     begin
@@ -229,7 +229,11 @@ class LogStash::Outputs::Analyticdb < LogStash::Outputs::Base
     end
 
     begin
-      events.each do |event|
+      pos = 0
+      insert_sql = ""
+      @logger.debug("events size: #{events.size}")
+      while pos < events.size do
+        event = events[pos]
         statement = connection.prepareStatement(
             (@unsafe_statement == true) ? event.sprintf(@statement[0]) : @statement[0]
         )
@@ -237,23 +241,45 @@ class LogStash::Outputs::Analyticdb < LogStash::Outputs::Base
           statement = add_statement_event_params(statement, event) if @statement.length > 1
           stmt_str = statement.toString
           one_sql = stmt_str[stmt_str.index(": ") + 2, stmt_str.length]
-          if sql_len + one_sql.length >= @commit_size
+          # on duplicate key start pos
+          on_duplicate_pos = one_sql.downcase.index(/on(\s+)duplicate/)
+          if on_duplicate_pos == nil
+            batch_insert_values_end_pos = one_sql.length
+          else
+            batch_insert_values_end_pos = on_duplicate_pos
+          end
+          @logger.debug("one_sql: #{one_sql}")
+          # trigger batch insert
+          if insert_sql.length + one_sql.length >= @commit_size
+            if insert_sql.length == 0
+              insert_sql = one_sql[0, batch_insert_values_end_pos]
+            end
+            if batch_insert_values_end_pos != one_sql.length
+              insert_sql.concat(one_sql[batch_insert_values_end_pos, one_sql.length - batch_insert_values_end_pos])
+            end
+            @logger.debug("batch 1 insert sql: #{insert_sql}")
             statement.execute(insert_sql)
-            sql_len = 0
             insert_sql = ""
           end
-          if sql_len == 0
-            insert_sql = one_sql
-            sql_len = one_sql.length
+          if insert_sql.length == 0
+            insert_sql = one_sql[0, batch_insert_values_end_pos]
           else
-            insert_sql.concat(",").concat(one_sql[@pre_len, one_sql.length])
-            sql_len = sql_len + one_sql.length - @pre_len
+            insert_sql = insert_sql.rstrip
+            insert_sql.concat(", ").concat(one_sql[@pre_len, batch_insert_values_end_pos - @pre_len])
           end
+          # loop to end
+          if pos == events.size - 1
+            if batch_insert_values_end_pos != one_sql.length
+              insert_sql.concat(one_sql[batch_insert_values_end_pos, one_sql.length - batch_insert_values_end_pos])
+            end
+            @logger.debug("batch 2 insert sql: #{insert_sql}")
+            statement.execute(insert_sql)
+          end
+          pos += 1
         rescue => e
           retry_exception?(e, event.to_json())
         end
       end
-      statement.execute(insert_sql)
     rescue => e
       @logger.error("Submit data error, sql is #{insert_sql}, error is #{e}")
       is_insert_err = true
@@ -306,8 +332,11 @@ class LogStash::Outputs::Analyticdb < LogStash::Outputs::Base
         attempts += 1
 
         if attempts > @max_flush_exceptions
-          @logger.error("JDBC - max_flush_exceptions has been reached. #{submit_actions.length} events have been unable to be sent to SQL and are being dropped. See previously logged exceptions for details.")
-          break
+          if (@skip_exception)
+            @logger.error("JDBC - max_flush_exceptions has been reached. #{submit_actions.length} events have been unable to be sent to SQL and are being skipped. See previously logged exceptions for details.")
+            break
+          end
+          raise "JDBC - max_flush_exceptions #{max_flush_exceptions} has been reached. #{submit_actions.length} events have been unable to be sent to SQL and are crashed. See previously logged exceptions for details."
         end
       end
 
@@ -370,10 +399,11 @@ class LogStash::Outputs::Analyticdb < LogStash::Outputs::Base
   end
 
   def retry_exception?(exception, event)
-    retrying = (exception.respond_to? 'getSQLState' and (RETRYABLE_SQLSTATE_CLASSES.include?(exception.getSQLState.to_s[0, 2]) or @retry_sql_states.include?(exception.getSQLState)))
+    # retrying = (exception.respond_to? 'getSQLState' and (RETRYABLE_SQLSTATE_CLASSES.include?(exception.getSQLState.to_s[0, 2]) or @retry_sql_states.include?(exception.getSQLState)))
+    retrying = true
     log_jdbc_exception(exception, retrying, event)
 
-    true 
+    retrying
   end
 
   def log_jdbc_exception(exception, retrying, event)
@@ -401,4 +431,4 @@ class LogStash::Outputs::Analyticdb < LogStash::Outputs::Base
     doubled = current_interval * 2
     doubled > @retry_max_interval ? @retry_max_interval : doubled
   end
-end # class LogStash::Outputs::analyticdb
+end # class LogStash::Outputs::analyticdb

data/logstash-output-analyticdb.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-analyticdb'
-  s.version = '5.4.0.6'
+  s.version = '5.4.0.11'
   s.licenses = ['Apache License (2.0)']
   s.summary = 'This plugin allows you to output to SQL, via JDBC'
   s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install 'logstash-output-analyticdb'. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-analyticdb
 version: !ruby/object:Gem::Version
-  version: 5.4.0.6
+  version: 5.4.0.11
 platform: ruby
 authors:
 - the_angry_angel
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-03-23 00:00:00.000000000 Z
+date: 2021-06-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -125,8 +125,8 @@ files:
 - vendor/jar-dependencies/runtime-jars/HikariCP-2.7.2.jar
 - vendor/jar-dependencies/runtime-jars/log4j-api-2.6.2.jar
 - vendor/jar-dependencies/runtime-jars/log4j-slf4j-impl-2.6.2.jar
-- vendor/jar-dependencies/runtime-jars/mysql-connector-java-5.1.36.jar
 - vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.25.jar
+- vendor/jar/jdbc/mysql-connector-java-8.0.11.jar
 homepage: https://github.com/wuchase/logstash-output-analyticdb
 licenses:
 - Apache License (2.0)
@@ -151,7 +151,7 @@ requirements:
 - jar 'com.zaxxer:HikariCP', '2.7.2'
 - jar 'org.apache.logging.log4j:log4j-slf4j-impl', '2.6.2'
 rubyforge_project:
-rubygems_version: 2.7.6
+rubygems_version: 2.7.10
 signing_key:
 specification_version: 4
 summary: This plugin allows you to output to SQL, via JDBC