fluent-plugin-sql 2.2.0 → 2.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 02d185dee05a4611437c046d0d279fb7299a874dd6c46a76d174a03c983b2f16
-  data.tar.gz: 71708e3128e2ec604ccccaf4ca7aeeb13c20853f2cdf0f72bbf4264125706ea2
+  metadata.gz: faedf22555693658bbb43e81ed55ec147f00196e883b2f53fe42cb3617860440
+  data.tar.gz: df4fa7f20a2bc8e779b6ed1fc80f42b70bc2be1e2e990c0ac9c9916a2ec02570
 SHA512:
-  metadata.gz: 94aff71abbd5fea54eca7b2d9534fe2615992586add9f2ced691770a21aae95e45c4276ed6b076c5505eca634d3ff82b1eb321a13959021791704fde88691533
-  data.tar.gz: ad2d4fe75de3032775ef25c6d6c5062a43ec01fc715a9a4e59371022975ae421c9e957282da0c8777a159f873b04280f09bd9aa34e2dc3fd512ac54136599ed6
+  metadata.gz: e5176135b712ac70afb866fe5f5139ddbc49f76c0180e28ed8a5cb37be5b109d0639a2b62963c0586119f6b0f55c78eb59501cb459af2afae9c1ee9a57edba15
+  data.tar.gz: e6644d47e0115f3030d42e089dbce1e70e964dc6b3c549e6f69a8aad18962ad4592685224de28554a4be1e1e6bb4da6917e06f496ab981eea735983755c39464

data/.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: 'github-actions'
+    directory: '/'
+    schedule:
+      interval: 'weekly'

data/.github/workflows/linux-ruby-head.yml

@@ -0,0 +1,45 @@
+name: Test with Ruby head
+on:
+  schedule:
+    - cron: '11 23 * * 0'
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: [ 'head' ]
+        postgres: [ '16', '15', '14', '13', '12' ]
+        os:
+          - ubuntu-latest
+    services:
+      postgres:
+        image: postgres:${{ matrix.postgres }}
+        ports:
+          - 5432:5432
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: password
+          POSTGRES_DB: test
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+    name: Ruby ${{ matrix.ruby }} with PostgreSQL ${{ matrix.postgres }} unit testing on ${{ matrix.os }}
+    steps:
+    - uses: actions/checkout@v4
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: ${{ matrix.ruby }}
+        bundler-cache: true
+    - name: prepare database
+      env:
+        PGPASSWORD: password
+      run: |
+        psql -h localhost -p 5432 -U postgres -c "CREATE ROLE fluentd WITH LOGIN ENCRYPTED PASSWORD 'fluentd';"
+        psql -h localhost -p 5432 -U postgres -c "CREATE DATABASE fluentd_test OWNER fluentd;"
+    - name: unit testing
+      run: bundle exec rake test

data/.github/workflows/linux.yml

@@ -1,41 +1,20 @@
-name: linux
+name: Test
 on:
-  - push
-  - pull_request
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
 
 jobs:
   build:
     runs-on: ${{ matrix.os }}
-    continue-on-error: ${{ matrix.experimental }}
     strategy:
       fail-fast: false
       matrix:
-        ruby: [ '2.5', '2.6', '2.7', '3.0' ]
-        postgres: [ '9', '10', '11', '12', '13' ]
+        ruby: [ '3.3', '3.2', '3.1', '3.0', '2.7' ]
+        postgres: [ '16', '15', '14', '13', '12' ]
         os:
           - ubuntu-latest
-        experimental: [false]
-        include:
-          - ruby: head
-            postgres: '9'
-            os: ubuntu-latest
-            experimental: true
-          - ruby: head
-            postgres: '10'
-            os: ubuntu-latest
-            experimental: true
-          - ruby: head
-            postgres: '11'
-            os: ubuntu-latest
-            experimental: true
-          - ruby: head
-            postgres: '12'
-            os: ubuntu-latest
-            experimental: true
-          - ruby: head
-            postgres: '13'
-            os: ubuntu-latest
-            experimental: true
     services:
       postgres:
         image: postgres:${{ matrix.postgres }}
@@ -52,10 +31,11 @@ jobs:
           --health-retries 5
     name: Ruby ${{ matrix.ruby }} with PostgreSQL ${{ matrix.postgres }} unit testing on ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{ matrix.ruby }}
+        bundler-cache: true
     - name: prepare database
       env:
         PGPASSWORD: password
@@ -63,7 +43,4 @@ jobs:
         psql -h localhost -p 5432 -U postgres -c "CREATE ROLE fluentd WITH LOGIN ENCRYPTED PASSWORD 'fluentd';"
         psql -h localhost -p 5432 -U postgres -c "CREATE DATABASE fluentd_test OWNER fluentd;"
     - name: unit testing
-      run: |
-        gem install bundler rake
-        bundle install --jobs 4 --retry 3
-        bundle exec rake test
+      run: bundle exec rake test

data/.gitignore

@@ -0,0 +1,4 @@
+Gemfile.lock
+pkg/*
+.vscode
+.idea/

data/ChangeLog

@@ -1,3 +1,11 @@
+Release 2.3.1 - 2024/02/29
+
+　* in_sql: Fix incompatibility against Ruby 3.2
+
+Release 2.3.0 - 2021/10/08
+
+　* in_sql: Make table time_format configurable
+
 Release 2.2.0 - 2021/04/20
 
 　* Upgrade ActiveRecord to 6.1

data/README.md

@@ -44,6 +44,7 @@ It stores last selected rows to a file (named *state\_file*) to not forget the l
       @type sql
 
       host rdb_host
+	  port rdb_port
       database rdb_database
       adapter mysql2_or_postgresql_or_etc
       username myusername
@@ -68,6 +69,7 @@ It stores last selected rows to a file (named *state\_file*) to not forget the l
         tag table2  # optional
         update_column updated_at
         time_column updated_at  # optional
+        time_format %Y-%m-%d %H:%M:%S.%6N # optional
       </table>
 
       # detects all tables instead of <table> sections
@@ -93,6 +95,7 @@ It stores last selected rows to a file (named *state\_file*) to not forget the l
 * **update_column**: see above description
 * **time_column** (optional): if this option is set, this plugin uses this column's value as the the event's time. Otherwise it uses current time.
 * **primary_key** (optional): if you want to get data from the table which doesn't have primary key like PostgreSQL's View, set this parameter.
+* **time_format** (optional): if you want to specify the format of the date used in the query, useful when using alternative adapters which have restrictions on format
 
 ## Input: Limitation
 

data/VERSION

@@ -1 +1 @@
-2.2.0
+2.3.1

data/lib/fluent/plugin/in_sql.rb

@@ -59,7 +59,7 @@ module Fluent::Plugin
       config_param :update_column, :string, default: nil
       config_param :time_column, :string, default: nil
       config_param :primary_key, :string, default: nil
-
+      config_param :time_format, :string, default: '%Y-%m-%d %H:%M:%S.%6N%z'
       attr_reader :log
 
       def configure(conf)
@@ -74,10 +74,13 @@ module Fluent::Plugin
         # creates a model for this table
         table_name = @table
         primary_key = @primary_key
+        time_format = @time_format
+
         @model = Class.new(base_model) do
           self.table_name = table_name
           self.inheritance_column = '_never_use_'
           self.primary_key = primary_key if primary_key
+          self.const_set(:TIME_FORMAT, time_format)
 
           #self.include_root_in_json = false
 
@@ -86,7 +89,7 @@ module Fluent::Plugin
             if v.respond_to?(:to_msgpack)
               v
             elsif v.is_a? Time
-              v.strftime('%Y-%m-%d %H:%M:%S.%6N%z')
+              v.strftime(self.class::TIME_FORMAT)
             else
               v.to_s
             end
@@ -279,7 +282,7 @@ module Fluent::Plugin
         require 'yaml'
 
         @path = path
-        if File.exists?(@path)
+        if File.exist?(@path)
           @data = YAML.load_file(@path)
           if @data == false || @data == []
             # this happens if an users created an empty file accidentally

data/test/helper.rb

@@ -1,6 +1,6 @@
 require "test/unit"
 require "test/unit/rr"
-require "test/unit/notify"
+require "test/unit/notify" unless ENV['CI']
 require "fluent/test"
 require "fluent/plugin/out_sql"
 require "fluent/plugin/in_sql"

data/test/plugin/test_in_sql_with_state_file.rb

@@ -0,0 +1,96 @@
+require "helper"
+require "fluent/test/driver/input"
+
+class SqlInputStateFileTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def teardown
+  end
+
+  CONFIG = %[
+    adapter postgresql
+    host localhost
+    port 5432
+    database fluentd_test
+
+    username fluentd
+    password fluentd
+
+    state_file /tmp/sql_state
+
+    schema_search_path public
+
+    tag_prefix db
+
+    <table>
+      table messages
+      tag logs
+      update_column updated_at
+      time_column updated_at
+    </table>
+  ]
+
+  def create_driver(conf = CONFIG)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::SQLInput).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver
+    expected = {
+      host: "localhost",
+      port: 5432,
+      adapter: "postgresql",
+      database: "fluentd_test",
+      username: "fluentd",
+      password: "fluentd",
+      schema_search_path: "public",
+      tag_prefix: "db"
+    }
+    actual = {
+      host: d.instance.host,
+      port: d.instance.port,
+      adapter: d.instance.adapter,
+      database: d.instance.database,
+      username: d.instance.username,
+      password: d.instance.password,
+      schema_search_path: d.instance.schema_search_path,
+      tag_prefix: d.instance.tag_prefix
+    }
+    assert_equal(expected, actual)
+    tables = d.instance.instance_variable_get(:@tables)
+    assert_equal(1, tables.size)
+    messages = tables.first
+    assert_equal("messages", messages.table)
+    assert_equal("logs", messages.tag)
+  end
+
+  def test_message
+    d = create_driver(CONFIG + "select_interval 1")
+    Message.create!(message: "message 1")
+    Message.create!(message: "message 2")
+    Message.create!(message: "message 3")
+
+    d.end_if do
+      d.record_count >= 3
+    end
+    d.run
+
+    assert_equal("db.logs", d.events[0][0])
+    expected = [
+      [d.events[0][1], "message 1"],
+      [d.events[1][1], "message 2"],
+      [d.events[2][1], "message 3"],
+    ]
+    actual = [
+      [Fluent::EventTime.parse(d.events[0][2]["updated_at"]), d.events[0][2]["message"]],
+      [Fluent::EventTime.parse(d.events[1][2]["updated_at"]), d.events[1][2]["message"]],
+      [Fluent::EventTime.parse(d.events[2][2]["updated_at"]), d.events[2][2]["message"]],
+    ]
+    assert_equal(expected, actual)
+  end
+
+  class Message < ActiveRecord::Base
+  end
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-sql
 version: !ruby/object:Gem::Version
-  version: 2.2.0
+  version: 2.3.1
 platform: ruby
 authors:
 - Sadayuki Furuhashi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-04-20 00:00:00.000000000 Z
+date: 2024-02-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -134,7 +134,10 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/dependabot.yml"
+- ".github/workflows/linux-ruby-head.yml"
 - ".github/workflows/linux.yml"
+- ".gitignore"
 - ChangeLog
 - Gemfile
 - Gemfile.v0.12
@@ -148,6 +151,7 @@ files:
 - test/helper.rb
 - test/plugin/test_in_sql.rb
 - test/plugin/test_in_sql_with_custom_time.rb
+- test/plugin/test_in_sql_with_state_file.rb
 - test/plugin/test_out_sql.rb
 homepage: https://github.com/fluent/fluent-plugin-sql
 licenses: