logstash-output-analyticdb 5.4.0.10 → 5.4.0.11

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1622874dc82c31b61bcbe5650256b56a09459e3f3acd0855c44328c7cd95737b
-  data.tar.gz: d29135a949ebccc68532f130027d635d0312c2d98c2a01f1cb03fcf6bab70a4b
+  metadata.gz: 27a21842e6bfba6088e5e56ece9763a7ef3288eb135ccf0689bd632d2784c676
+  data.tar.gz: 772b83405d4981e98bb8685490591ee100a6a4f9c02a55b98e0635c365f4dc26
 SHA512:
-  metadata.gz: fa7bdb92f436cbedfb2f5c655fb26a799ddb260e9ca8a4bd05a550922226cafe96b68205e62f7ff57f48bc97d6eecaca2ae1e64128ec05708ae3402a64281580
-  data.tar.gz: 1b1189e08497ce56ab1370df1e6d427d5482cbc38eb355052464fc99cac50cf39862f1b82baafe920e6ee2ed7b90517222a826d04b3cf885e7ad87a3fbf6d05b
+  metadata.gz: b41bb657102f6a2512ac3f503187f55f42219db01ee9fd0f3f2d7e6bfe93d35d70aff2a1634e7908d58d8a72256ab8994e2074b1ce5a3609228ba5a0a9fbab25
+  data.tar.gz: 725a27df6db012727913bced51dba68a7c2fdb642483f637a5fdbc85be19014b7a0d825b22ef5cbb5c7bf24e04ab5d4b38e485fcfa1e7e3b566576318e10029d

data/README.md

@@ -6,14 +6,6 @@ This plugin is provided as an external plugin and is not part of the Logstash pr
 This plugin allows you to output to AnalyticDB database, using JDBC adapters.
 See below for tested adapters, and example configurations.
 
-## Support & release schedule
-I no longer have time at work to maintain this plugin in step with Logstash's releases, and I am not completely immersed in the Logstash ecosystem. If something is broken for you I will do my best to help, but I cannot guarantee timeframes.
-
-Pull requests are always welcome. 
-
-## Changelog
-See CHANGELOG.md
-
 ## Versions
 Released versions are available via rubygems, and typically tagged.
 

data/lib/logstash/outputs/analyticdb.rb

@@ -217,8 +217,6 @@ class LogStash::Outputs::Analyticdb < LogStash::Outputs::Base
     connection = nil
     statement = nil
     events_to_retry = []
-    insert_sql = ""
-    sql_len = 0
     is_insert_err = false
 
     begin
@@ -231,7 +229,11 @@ class LogStash::Outputs::Analyticdb < LogStash::Outputs::Base
     end
 
     begin
-      events.each do |event|
+      pos = 0
+      insert_sql = ""
+      @logger.debug("events size: #{events.size}")
+      while pos < events.size do
+        event = events[pos]
         statement = connection.prepareStatement(
             (@unsafe_statement == true) ? event.sprintf(@statement[0]) : @statement[0]
         )
@@ -239,24 +241,45 @@ class LogStash::Outputs::Analyticdb < LogStash::Outputs::Base
           statement = add_statement_event_params(statement, event) if @statement.length > 1
           stmt_str = statement.toString
           one_sql = stmt_str[stmt_str.index(": ") + 2, stmt_str.length]
-          if sql_len + one_sql.length >= @commit_size
+          # on duplicate key start pos
+          on_duplicate_pos = one_sql.downcase.index(/on(\s+)duplicate/)
+          if on_duplicate_pos == nil
+            batch_insert_values_end_pos = one_sql.length
+          else
+            batch_insert_values_end_pos = on_duplicate_pos
+          end
+          @logger.debug("one_sql: #{one_sql}")
+          # trigger batch insert
+          if insert_sql.length + one_sql.length >= @commit_size
+            if insert_sql.length == 0
+              insert_sql = one_sql[0, batch_insert_values_end_pos]
+            end
+            if batch_insert_values_end_pos != one_sql.length
+              insert_sql.concat(one_sql[batch_insert_values_end_pos, one_sql.length - batch_insert_values_end_pos])
+            end
+            @logger.debug("batch 1 insert sql: #{insert_sql}")
             statement.execute(insert_sql)
-            sql_len = 0
             insert_sql = ""
           end
-          if sql_len == 0
-            insert_sql = one_sql
-            sql_len = one_sql.length
+          if insert_sql.length == 0
+            insert_sql = one_sql[0, batch_insert_values_end_pos]
           else
-            insert_sql.concat(",").concat(one_sql[@pre_len, one_sql.length])
-            sql_len = sql_len + one_sql.length - @pre_len
+            insert_sql = insert_sql.rstrip
+            insert_sql.concat(", ").concat(one_sql[@pre_len, batch_insert_values_end_pos - @pre_len])
+          end
+          # loop to end
+          if pos == events.size - 1
+            if batch_insert_values_end_pos != one_sql.length
+              insert_sql.concat(one_sql[batch_insert_values_end_pos, one_sql.length - batch_insert_values_end_pos])
+            end
+            @logger.debug("batch 2 insert sql: #{insert_sql}")
+            statement.execute(insert_sql)
           end
+          pos += 1
         rescue => e
           retry_exception?(e, event.to_json())
-          is_insert_err = true
         end
       end
-      statement.execute(insert_sql)
     rescue => e
       @logger.error("Submit data error, sql is #{insert_sql}, error is #{e}")
       is_insert_err = true
@@ -313,8 +336,7 @@ class LogStash::Outputs::Analyticdb < LogStash::Outputs::Base
             @logger.error("JDBC - max_flush_exceptions has been reached. #{submit_actions.length} events have been unable to be sent to SQL and are being skipped. See previously logged exceptions for details.")
             break
           end
-          raise "JDBC - max_flush_exceptions #{max_flush_exceptions} has been reached. #{submit_actions.length} events have been unable to be sent to SQL and are being dropped. See previously logged exceptions for details."
-          break
+          raise "JDBC - max_flush_exceptions #{max_flush_exceptions} has been reached. #{submit_actions.length} events have been unable to be sent to SQL and are crashed. See previously logged exceptions for details."
         end
       end
 
@@ -409,4 +431,4 @@ class LogStash::Outputs::Analyticdb < LogStash::Outputs::Base
     doubled = current_interval * 2
     doubled > @retry_max_interval ? @retry_max_interval : doubled
   end
-end # class LogStash::Outputs::analyticdb
+end # class LogStash::Outputs::analyticdb

data/logstash-output-analyticdb.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-analyticdb'
-  s.version = '5.4.0.10'
+  s.version = '5.4.0.11'
   s.licenses = ['Apache License (2.0)']
   s.summary = 'This plugin allows you to output to SQL, via JDBC'
   s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install 'logstash-output-analyticdb'. This gem is not a stand-alone program"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-analyticdb
 version: !ruby/object:Gem::Version
-  version: 5.4.0.10
+  version: 5.4.0.11
 platform: ruby
 authors:
 - the_angry_angel
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-03-25 00:00:00.000000000 Z
+date: 2021-06-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -125,8 +125,8 @@ files:
 - vendor/jar-dependencies/runtime-jars/HikariCP-2.7.2.jar
 - vendor/jar-dependencies/runtime-jars/log4j-api-2.6.2.jar
 - vendor/jar-dependencies/runtime-jars/log4j-slf4j-impl-2.6.2.jar
-- vendor/jar-dependencies/runtime-jars/mysql-connector-java-5.1.36.jar
 - vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.25.jar
+- vendor/jar/jdbc/mysql-connector-java-8.0.11.jar
 homepage: https://github.com/wuchase/logstash-output-analyticdb
 licenses:
 - Apache License (2.0)
@@ -151,7 +151,7 @@ requirements:
 - jar 'com.zaxxer:HikariCP', '2.7.2'
 - jar 'org.apache.logging.log4j:log4j-slf4j-impl', '2.6.2'
 rubyforge_project:
-rubygems_version: 2.7.6
+rubygems_version: 2.7.10
 signing_key:
 specification_version: 4
 summary: This plugin allows you to output to SQL, via JDBC