fluent-plugin-sql 2.3.2 → 2.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8d90c196fbc84a7921ba99ea09cdc7aef2c564b4c19799cc1bcce9973e1f391e
-  data.tar.gz: 358dbde32a842e26b2ff9e34a11acff25790624dec8ae1e4689aaa8488d179de
+  metadata.gz: 197db6803c87ca64448c432656a89d879f8e465258928e697d487ad16fd462b6
+  data.tar.gz: 89a4f1866a939aaec3122ba8259f4d9a74d851175743c117984ab2e29cc757bf
 SHA512:
-  metadata.gz: 1cbe495ae7aab0133427155b1a136f8a499b9b0924305bf24807c0f2950f2a84b7a3d0657093375e682be37d2d81f96b503e77635c7ddb24f51a5abb40914a51
-  data.tar.gz: 9b856ce7892a9d9f2fd35d36137eca67508d129194043516e976f5497518da251604c35cba15d483386d0879bb073e2fe9fd1d96a0e027dcbe66fb5fd2a78b7d
+  metadata.gz: b22a4fefd2608b511a17247e92ee07ef475a570c9f1b17e713156ced301b69d51ff994d2e2ed39832134241227fa24fd381fe792b97ccf38cf212ccc01d9f9b4
+  data.tar.gz: 00eae6b85e603667715d5ddd72f788dd1c8614cd8b0cb5c31d7d515cdba8f2aec3d84b39f12c8c6a6e65951c42c6d13189be54129a8d1022d0b333165c87ee8d

data/.github/dependabot.yml

@@ -3,4 +3,4 @@ updates:
   - package-ecosystem: 'github-actions'
     directory: '/'
     schedule:
-      interval: 'weekly'
+      interval: 'monthly'

data/.github/workflows/linux-ruby-head.yml

@@ -11,7 +11,7 @@ jobs:
       fail-fast: false
       matrix:
         ruby: [ 'head' ]
-        postgres: [ '16', '15', '14', '13', '12' ]
+        postgres: [ '18', '17', '16', '15', '14' ]
         os:
           - ubuntu-latest
     services:
@@ -30,8 +30,8 @@ jobs:
           --health-retries 5
     name: Ruby ${{ matrix.ruby }} with PostgreSQL ${{ matrix.postgres }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v6
-    - uses: ruby/setup-ruby@v1
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+    - uses: ruby/setup-ruby@c4e5b1316158f92e3d49443a9d58b31d25ac0f8f # v1.306.0
       with:
         ruby-version: ${{ matrix.ruby }}
         bundler-cache: true

data/.github/workflows/linux.yml

@@ -4,15 +4,24 @@ on:
     branches: [master]
   pull_request:
     branches: [master]
-
+  schedule:
+    - cron: '0 0 1 * *'
 jobs:
+  ruby-versions:
+    uses: ruby/actions/.github/workflows/ruby_versions.yml@master
+    with:
+      engine: cruby
+      min_version: 2.7
   build:
+    needs: ruby-versions
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '4.0', '3.4', '3.3', '3.2', '3.1', '3.0', '2.7' ]
-        postgres: [ '16', '15', '14', '13', '12' ]
+        ruby: ${{ fromJson(needs.ruby-versions.outputs.versions) }}
+        exclude:
+          - ruby: head
+        postgres: [ '18', '17', '16', '15', '14' ]
         os:
           - ubuntu-latest
     services:
@@ -31,8 +40,8 @@ jobs:
           --health-retries 5
     name: Ruby ${{ matrix.ruby }} with PostgreSQL ${{ matrix.postgres }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v6
-    - uses: ruby/setup-ruby@v1
+    - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+    - uses: ruby/setup-ruby@c4e5b1316158f92e3d49443a9d58b31d25ac0f8f # v1.306.0
       with:
         ruby-version: ${{ matrix.ruby }}
         bundler-cache: true

data/ChangeLog

@@ -1,3 +1,7 @@
+Release 2.3.3 - 2026/05/18
+
+  * out_sql: fix unmapped columns being explicitly set to NULL on INSERT (#169)
+
 Release 2.3.2 - 2026/01/13
 
   * out_sql: Use isolated connection configurations in the output plugin (#152)

data/VERSION

@@ -1 +1 @@
-2.3.2
+2.3.3

data/lib/fluent/plugin/out_sql.rb

@@ -90,23 +90,26 @@ module Fluent::Plugin
 
       def import(chunk, output)
         tag = chunk.metadata.tag
+        columns = []
         records = []
         chunk.msgpack_each { |time, data|
           begin
             data = output.inject_values_to_record(tag, time, data)
-            records << @model.new(@format_proc.call(data))
+            new_record = @format_proc.call(data)
+            columns |= new_record.keys
+            records << @model.new(new_record)
           rescue => e
             args = {error: e, table: @table, record: Yajl.dump(data)}
             @log.warn "Failed to create the model. Ignore a record:", args
           end
         }
         begin
-          @model.import(records)
+          @model.import(columns, records)
         rescue ActiveRecord::StatementInvalid, ActiveRecord::Import::MissingColumnError => e
           if @enable_fallback
             # ignore other exceptions to use Fluentd retry mechanizm
             @log.warn "Got deterministic error. Fallback to one-by-one import", error: e
-            one_by_one_import(records)
+            one_by_one_import(columns, records)
           else
             @log.warn "Got deterministic error. Fallback is disabled", error: e
             raise e
@@ -114,11 +117,11 @@ module Fluent::Plugin
         end
       end
 
-      def one_by_one_import(records)
+      def one_by_one_import(columns, records)
         records.each { |record|
           retries = 0
           begin
-            @model.import([record])
+            @model.import(columns, [record])
           rescue ActiveRecord::StatementInvalid, ActiveRecord::Import::MissingColumnError => e
             @log.error "Got deterministic error again. Dump a record", error: e, record: record
           rescue => e

data/test/fixtures/schema.rb

@@ -27,5 +27,9 @@ ActiveRecord::Schema.define(version: 20160225030107) do
     t.datetime "updated_at",  null: false
     t.string "custom_time"
   end
-end
 
+  create_table "messages_unmapped", force: :cascade do |t|
+    t.string "message"
+    t.string "status", default: 'default_status'
+  end
+end

data/test/plugin/test_out_sql.rb

@@ -80,6 +80,41 @@ class SqlOutputTest < Test::Unit::TestCase
     assert_equal(["message1", "message2"], messages)
   end
 
+  def test_unmapped_column_uses_db_default
+    config = %[
+      host localhost
+      port 5432
+      adapter postgresql
+
+      database fluentd_test
+      username fluentd
+      password fluentd
+
+      schema_search_path public
+
+      remove_tag_prefix db
+
+      <table>
+        table messages_unmapped
+        column_mapping message:message
+      </table>
+    ]
+
+    d = create_driver(config)
+    time = Time.parse("2011-01-02 13:14:15 UTC").to_i
+
+    d.run(default_tag: 'test') do
+      d.feed(time, {'message' => 'hello'})
+    end
+
+    records = ActiveRecord::Base.connection.select_all("SELECT * FROM messages_unmapped").to_a
+
+    assert_equal 1, records.size
+    assert_equal 'hello', records.first['message']
+    # The 'status' column is not mapped, so it should use the default value defined in the database schema
+    assert_equal 'default_status', records.first['status']
+  end
+
   class Fallback < self
     def test_simple
       d = create_driver
@@ -91,10 +126,10 @@ class SqlOutputTest < Test::Unit::TestCase
 
         default_table = d.instance.instance_variable_get(:@default_table)
         model = default_table.instance_variable_get(:@model)
-        mock(model).import(anything).at_least(1) do
+        mock(model).import(anything, anything).at_least(1) do
           raise ActiveRecord::Import::MissingColumnError.new("dummy_table", "dummy_column")
         end
-        mock(default_table).one_by_one_import(anything)
+        mock(default_table).one_by_one_import(anything, anything)
       end
     end
 
@@ -108,10 +143,10 @@ class SqlOutputTest < Test::Unit::TestCase
 
         default_table = d.instance.instance_variable_get(:@default_table)
         model = default_table.instance_variable_get(:@model)
-        mock(model).import([anything, anything]).once do
+        mock(model).import(anything, [anything, anything]).once do
           raise ActiveRecord::Import::MissingColumnError.new("dummy_table", "dummy_column")
         end
-        mock(model).import([anything]).times(12) do
+        mock(model).import(anything, [anything]).times(12) do
           raise StandardError
         end
         assert_equal(5, default_table.instance_variable_get(:@num_retries))

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-sql
 version: !ruby/object:Gem::Version
-  version: 2.3.2
+  version: 2.3.3
 platform: ruby
 authors:
 - Sadayuki Furuhashi
@@ -190,7 +190,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.9
+rubygems_version: 4.0.10
 specification_version: 4
 summary: SQL input/output plugin for Fluentd event collector
 test_files: