fluent-plugin-sql 2.3.1 → 2.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: faedf22555693658bbb43e81ed55ec147f00196e883b2f53fe42cb3617860440
-  data.tar.gz: df4fa7f20a2bc8e779b6ed1fc80f42b70bc2be1e2e990c0ac9c9916a2ec02570
+  metadata.gz: 197db6803c87ca64448c432656a89d879f8e465258928e697d487ad16fd462b6
+  data.tar.gz: 89a4f1866a939aaec3122ba8259f4d9a74d851175743c117984ab2e29cc757bf
 SHA512:
-  metadata.gz: e5176135b712ac70afb866fe5f5139ddbc49f76c0180e28ed8a5cb37be5b109d0639a2b62963c0586119f6b0f55c78eb59501cb459af2afae9c1ee9a57edba15
-  data.tar.gz: e6644d47e0115f3030d42e089dbce1e70e964dc6b3c549e6f69a8aad18962ad4592685224de28554a4be1e1e6bb4da6917e06f496ab981eea735983755c39464
+  metadata.gz: b22a4fefd2608b511a17247e92ee07ef475a570c9f1b17e713156ced301b69d51ff994d2e2ed39832134241227fa24fd381fe792b97ccf38cf212ccc01d9f9b4
+  data.tar.gz: 00eae6b85e603667715d5ddd72f788dd1c8614cd8b0cb5c31d7d515cdba8f2aec3d84b39f12c8c6a6e65951c42c6d13189be54129a8d1022d0b333165c87ee8d

data/.github/dependabot.yml

@@ -3,4 +3,4 @@ updates:
   - package-ecosystem: 'github-actions'
     directory: '/'
     schedule:
-      interval: 'weekly'
+      interval: 'monthly'

data/.github/workflows/linux-ruby-head.yml

@@ -11,7 +11,7 @@ jobs:
       fail-fast: false
       matrix:
         ruby: [ 'head' ]
-        postgres: [ '16', '15', '14', '13', '12' ]
+        postgres: [ '18', '17', '16', '15', '14' ]
         os:
           - ubuntu-latest
     services:
@@ -30,8 +30,8 @@ jobs:
           --health-retries 5
     name: Ruby ${{ matrix.ruby }} with PostgreSQL ${{ matrix.postgres }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v4
-    - uses: ruby/setup-ruby@v1
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+    - uses: ruby/setup-ruby@c4e5b1316158f92e3d49443a9d58b31d25ac0f8f # v1.306.0
       with:
         ruby-version: ${{ matrix.ruby }}
         bundler-cache: true

data/.github/workflows/linux.yml

@@ -4,15 +4,24 @@ on:
     branches: [master]
   pull_request:
     branches: [master]
-
+  schedule:
+    - cron: '0 0 1 * *'
 jobs:
+  ruby-versions:
+    uses: ruby/actions/.github/workflows/ruby_versions.yml@master
+    with:
+      engine: cruby
+      min_version: 2.7
   build:
+    needs: ruby-versions
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '3.3', '3.2', '3.1', '3.0', '2.7' ]
-        postgres: [ '16', '15', '14', '13', '12' ]
+        ruby: ${{ fromJson(needs.ruby-versions.outputs.versions) }}
+        exclude:
+          - ruby: head
+        postgres: [ '18', '17', '16', '15', '14' ]
         os:
           - ubuntu-latest
     services:
@@ -31,8 +40,8 @@ jobs:
           --health-retries 5
     name: Ruby ${{ matrix.ruby }} with PostgreSQL ${{ matrix.postgres }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v4
-    - uses: ruby/setup-ruby@v1
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+    - uses: ruby/setup-ruby@c4e5b1316158f92e3d49443a9d58b31d25ac0f8f # v1.306.0
       with:
         ruby-version: ${{ matrix.ruby }}
         bundler-cache: true

data/ChangeLog

@@ -1,3 +1,13 @@
+Release 2.3.3 - 2026/05/18
+
+  * out_sql: fix unmapped columns being explicitly set to NULL on INSERT (#169)
+
+Release 2.3.2 - 2026/01/13
+
+  * out_sql: Use isolated connection configurations in the output plugin (#152)
+  * gem: Add ostruct as dependency for Ruby 4.0 (#159)
+  * gem: Relax required activesupport version to accept latest one (#161, #148)
+
 Release 2.3.1 - 2024/02/29
 
 　* in_sql: Fix incompatibility against Ruby 3.2

data/VERSION

@@ -1 +1 @@
-2.3.1
+2.3.3

data/fluent-plugin-sql.gemspec

@@ -17,7 +17,8 @@ Gem::Specification.new do |gem|
   gem.license = "Apache-2.0"
 
   gem.add_dependency "fluentd", [">= 1.7.0", "< 2"]
-  gem.add_dependency 'activerecord', "~> 6.1"
+  gem.add_dependency "ostruct", '~> 0.6'
+  gem.add_dependency 'activerecord', [">= 6.1", "< 8.2"]
   gem.add_dependency 'activerecord-import', "~> 1.0"
   gem.add_development_dependency "rake", ">= 0.9.2"
   gem.add_development_dependency "test-unit", "> 3.1.0"

data/lib/fluent/plugin/out_sql.rb

@@ -90,23 +90,26 @@ module Fluent::Plugin
 
       def import(chunk, output)
         tag = chunk.metadata.tag
+        columns = []
         records = []
         chunk.msgpack_each { |time, data|
           begin
             data = output.inject_values_to_record(tag, time, data)
-            records << @model.new(@format_proc.call(data))
+            new_record = @format_proc.call(data)
+            columns |= new_record.keys
+            records << @model.new(new_record)
           rescue => e
             args = {error: e, table: @table, record: Yajl.dump(data)}
             @log.warn "Failed to create the model. Ignore a record:", args
           end
         }
         begin
-          @model.import(records)
+          @model.import(columns, records)
         rescue ActiveRecord::StatementInvalid, ActiveRecord::Import::MissingColumnError => e
           if @enable_fallback
             # ignore other exceptions to use Fluentd retry mechanizm
             @log.warn "Got deterministic error. Fallback to one-by-one import", error: e
-            one_by_one_import(records)
+            one_by_one_import(columns, records)
           else
             @log.warn "Got deterministic error. Fallback is disabled", error: e
             raise e
@@ -114,11 +117,11 @@ module Fluent::Plugin
         end
       end
 
-      def one_by_one_import(records)
+      def one_by_one_import(columns, records)
         records.each { |record|
           retries = 0
           begin
-            @model.import([record])
+            @model.import(columns, [record])
           rescue ActiveRecord::StatementInvalid, ActiveRecord::Import::MissingColumnError => e
             @log.error "Got deterministic error again. Dump a record", error: e, record: record
           rescue => e
@@ -206,7 +209,7 @@ module Fluent::Plugin
       end
 
       SQLOutput.const_set("BaseModel_#{rand(1 << 31)}", @base_model)
-      ActiveRecord::Base.establish_connection(config)
+      @base_model.establish_connection(config)
 
       # ignore tables if TableElement#init failed
       @tables.reject! do |te|
@@ -224,7 +227,7 @@ module Fluent::Plugin
     end
 
     def write(chunk)
-      ActiveRecord::Base.connection_pool.with_connection do
+      @base_model.connection_pool.with_connection do
 
         @tables.each { |table|
           tag = format_tag(chunk.metadata.tag)

data/test/fixtures/schema.rb

@@ -27,5 +27,9 @@ ActiveRecord::Schema.define(version: 20160225030107) do
     t.datetime "updated_at",  null: false
     t.string "custom_time"
   end
-end
 
+  create_table "messages_unmapped", force: :cascade do |t|
+    t.string "message"
+    t.string "status", default: 'default_status'
+  end
+end

data/test/plugin/test_out_sql.rb

@@ -80,6 +80,41 @@ class SqlOutputTest < Test::Unit::TestCase
     assert_equal(["message1", "message2"], messages)
   end
 
+  def test_unmapped_column_uses_db_default
+    config = %[
+      host localhost
+      port 5432
+      adapter postgresql
+
+      database fluentd_test
+      username fluentd
+      password fluentd
+
+      schema_search_path public
+
+      remove_tag_prefix db
+
+      <table>
+        table messages_unmapped
+        column_mapping message:message
+      </table>
+    ]
+
+    d = create_driver(config)
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.run(default_tag: 'test') do
+      d.feed(time, {'message' => 'hello'})
+    end
+
+    records = ActiveRecord::Base.connection.select_all("SELECT * FROM messages_unmapped").to_a
+
+    assert_equal 1, records.size
+    assert_equal 'hello', records.first['message']
+    # The 'status' column is not mapped, so it should use the default value defined in the database schema
+    assert_equal 'default_status', records.first['status']
+  end
+
   class Fallback < self
     def test_simple
       d = create_driver
@@ -91,10 +126,10 @@ class SqlOutputTest < Test::Unit::TestCase
 
         default_table = d.instance.instance_variable_get(:@default_table)
         model = default_table.instance_variable_get(:@model)
-        mock(model).import(anything).at_least(1) do
+        mock(model).import(anything, anything).at_least(1) do
           raise ActiveRecord::Import::MissingColumnError.new("dummy_table", "dummy_column")
         end
-        mock(default_table).one_by_one_import(anything)
+        mock(default_table).one_by_one_import(anything, anything)
       end
     end
 
@@ -108,10 +143,10 @@ class SqlOutputTest < Test::Unit::TestCase
 
         default_table = d.instance.instance_variable_get(:@default_table)
         model = default_table.instance_variable_get(:@model)
-        mock(model).import([anything, anything]).once do
+        mock(model).import(anything, [anything, anything]).once do
           raise ActiveRecord::Import::MissingColumnError.new("dummy_table", "dummy_column")
         end
-        mock(model).import([anything]).times(12) do
+        mock(model).import(anything, [anything]).times(12) do
           raise StandardError
         end
         assert_equal(5, default_table.instance_variable_get(:@num_retries))

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-sql
 version: !ruby/object:Gem::Version
-  version: 2.3.1
+  version: 2.3.3
 platform: ruby
 authors:
 - Sadayuki Furuhashi
-autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-02-29 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -31,19 +30,39 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2'
 - !ruby/object:Gem::Dependency
-  name: activerecord
+  name: ostruct
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '6.1'
+        version: '0.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.6'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
       - !ruby/object:Gem::Version
         version: '6.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '8.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '6.1'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '8.2'
 - !ruby/object:Gem::Dependency
   name: activerecord-import
   requirement: !ruby/object:Gem::Requirement
@@ -157,7 +176,6 @@ homepage: https://github.com/fluent/fluent-plugin-sql
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -172,8 +190,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
-signing_key: 
+rubygems_version: 4.0.10
 specification_version: 4
 summary: SQL input/output plugin for Fluentd event collector
-test_files: []
+test_files:
+- test/fixtures/schema.rb
+- test/helper.rb
+- test/plugin/test_in_sql.rb
+- test/plugin/test_in_sql_with_custom_time.rb
+- test/plugin/test_in_sql_with_state_file.rb
+- test/plugin/test_out_sql.rb