fluent-plugin-postgresql-csvlog 0.8.0 → 0.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 47a1cb4850459062da5931ab99d4180d53caed98985096be75a485a688e386c1
-  data.tar.gz: 3d529ac068bed5cebed3e8ac49bcfb2f6f1e9d2d4ea9427582d1bdfc26ae22d3
+  metadata.gz: 31b92e4a5c795b777632cb72a3981df1248564d3446133536dabedf4b5cce541
+  data.tar.gz: f79600c2b56f20744b8afeda9b1461d51e8dacd3901be9273cdb8a16cd71dd21
 SHA512:
-  metadata.gz: 927aba45b777040f1c5cb8b1e2044b0c2b8aca3b8590f1a7499285b835adfd15f276ef3ca3740193da61925beb22e0374314c87c482983007d2dd6cec5910ae0
-  data.tar.gz: 6e75b2674e449122ad5c991a7a39429c0f0bfeff0de37d7b059673f964baccbf8ba2249680d529332b3a40f2e369627d1813de378f9774fc07370cd884748749
+  metadata.gz: 0024223f3d954f0c3d7fe5789c23ad08ff901e4225a2ee8a23de1a85bb09330b8b26241c39c23853ec6c3c50d2dd0a655add67f24c551156a497c83d55219cb7
+  data.tar.gz: ea9eaa01530670cc3d7c38f370854bdb573dbd1d4cfaa41bc55c2ddefede29e2cf0d8ff7ba33af60fba7c4781afcd26eaa89fb99dade750d0da5902c974476e8

data/.gitlab-ci.yml

@@ -1,16 +1,32 @@
-image: "ruby:2.7"
+image: "ruby:${RUBY_VERSION}"
+
+variables:
+  RUBY_VERSION: "3.2"
+
+stages:
+  - test
+  - deploy
+
+include:
+  - component: gitlab.com/gitlab-org/components/gem-release/gem-release@~latest
+    inputs:
+      smoke_test_script: "ruby -r 'fluent/plugin/filter_postgresql_slowlog'"
+      file_pattern_to_trigger_release: "fluent-plugin-postgresql-csvlog.gemspec"
 
 test:
+  stage: test
   before_script:
     - bundle config set path vendor
     - bundle install --jobs $(nproc)
   script:
     - bundle exec rake test
   cache:
+    key: ruby-${RUBY_VERSION}
     paths:
       - vendor/ruby
 
 .iteration_test:
+  stage: test
   services:
     - name: postgres:$POSTGRES_SERVER_VERSION
       alias: postgres
@@ -27,25 +43,15 @@ test:
     paths:
       - vendor/ruby
 
-# integration tests for postgres 12
-itest_pg12:
-  extends: .iteration_test
-  variables:
-    POSTGRES_SERVER_VERSION: 12
-
-# integration tests for postgres 13
-itest_pg13:
+# integration tests for PostgreSQL
+itest:
   extends: .iteration_test
-  variables:
-    POSTGRES_SERVER_VERSION: 13
-
-# integration tests for postgres 14
-itest_pg14:
-  extends: .iteration_test
-  variables:
-    POSTGRES_SERVER_VERSION: 14
+  parallel:
+    matrix:
+      - POSTGRES_SERVER_VERSION: ["12", "13", "14", "15", "16"]
 
 end_to_end_verification_test:
+  stage: test
   image: docker:20.10.16
   services:
     - docker:20.10.16-dind

data/fluent-plugin-postgresql-csvlog.gemspec

@@ -2,7 +2,7 @@ $:.push File.expand_path('lib', __dir__)
 
 Gem::Specification.new do |s|
   s.name        = 'fluent-plugin-postgresql-csvlog'
-  s.version     = '0.8.0'
+  s.version     = '0.9.0'
   s.authors     = ['stanhu']
   s.email       = ['stanhu@gmail.com']
   s.homepage    = 'https://gitlab.com/gitlab-org/fluent-plugins/fluent-plugin-postgresql-csvlog'
@@ -15,8 +15,8 @@ Gem::Specification.new do |s|
   s.require_paths = ['lib']
 
   s.add_dependency 'fluentd', ['>= 1.0', '< 2']
-  s.add_dependency 'pg', '~> 1.1'
-  s.add_dependency 'pg_query', '~> 2.0'
+  s.add_dependency 'pg', '~> 1.5'
+  s.add_dependency 'pg_query', '~> 5.0'
 
   s.add_development_dependency 'rake'
   s.add_development_dependency 'test-unit', '~> 3.2'

data/lib/fluent/plugin/marginalia_extractor.rb

@@ -6,8 +6,8 @@ module Fluent::Plugin
   # MarginaliaExtractor provides the parse_marginalia_into_record
   # utility method, useful for extracting marginalia into fluentd records
   module MarginaliaExtractor
-    MARGINALIA_PREPENDED_REGEXP = %r{^(?<comment>/\*.*\*/)(?<sql>.*)}m.freeze
-    MARGINALIA_APPENDED_REGEXP = %r{(?<sql>.*)(?<comment>/\*.*\*/)\s*;?\s*$}m.freeze
+    MARGINALIA_PREPENDED_REGEXP = %r{^(?<comment>/\*.*?\*/)\s*(?<sql>.*)}m
+    MARGINALIA_APPENDED_REGEXP = %r{(?<sql>.*)(?<comment>/\*.*\*/)\s*;?\s*$}m
 
     # Injects marginalia into a fluentd record
     def parse_marginalia_into_record(record, key, strip_comment)

data/test/plugin/test_in_pg_stat_statements.rb

@@ -44,7 +44,7 @@ class PgStatStatementsInputTest < Test::Unit::TestCase
       d = create_driver
       row = {
         'queryid' => '1234',
-        'query' => 'SELECT * FROM users WHERE user_id = ?',
+        'query' => 'SELECT * FROM users WHERE user_id = 123',
         'calls' => 22,
         'rows' => 333,
         'total_time' => 44.44
@@ -54,8 +54,8 @@ class PgStatStatementsInputTest < Test::Unit::TestCase
 
       expected = {
         'fingerprint' => 'c071dee80d466e7d',
-        'query' => 'SELECT * FROM users WHERE user_id = ?',
-        'query_length' => 37,
+        'query' => 'SELECT * FROM users WHERE user_id = $1',
+        'query_length' => 39,
         'queryid' => '1234',
         'calls' => 22,
         'rows' => 333,

data/test/plugin/test_marginalia_extractor.rb

@@ -48,6 +48,19 @@ class Marginalia < Test::Unit::TestCase
     test_parse(sql, {}, 'sql', true, expected)
   end
 
+  test 'marginalia appended with other comments in SQL' do
+    sql = %(INSERT INTO "p_ci_builds_metadata" ("project_id", "config_options", "config_variables", "has_exposed_artifacts", "build_id", "partition_id") VALUES (40163635, '{"image":{"name":"$AWS_AMAZONLINUX_IMAGE"},"artifacts":{"paths":["${ASSET_NAME}/out"]},"before_script":["yum install -y python3 python3-pip zip findutils","cd ${ASSET_NAME}"],"script":["mkdir -p venv","cd venv","python3 -m venv pyspark_venvsource","source pyspark_venvsource/bin/activate","pip3 install -r ../scripts/requirements.txt","pip3 install venv-pack","venv-pack -f -o pyspark_venv.tar.gz","cd ..; mkdir -p out/lib","cd src/step/; cp -r --parents ./*/*main*.py ../../out"],"retry":{"max":2}}', '[{"key":"ASSET_NAME","value":"data-pipeline"}]', FALSE, 6269937106, 101) RETURNING "id" /*application:sidekiq,correlation_id:01HQNQAHYZF4Q294P59R7RP71H,jid:047ecd6920dc27ccd49b0aef,endpoint_id:PostReceive,db_config_name:ci*/)
+    expected = {
+      'application' => 'sidekiq',
+      'correlation_id' => '01HQNQAHYZF4Q294P59R7RP71H',
+      'jid' => '047ecd6920dc27ccd49b0aef',
+      'db_config_name' => 'ci',
+      'endpoint_id' => 'PostReceive',
+      'sql' => %(INSERT INTO "p_ci_builds_metadata" ("project_id", "config_options", "config_variables", "has_exposed_artifacts", "build_id", "partition_id") VALUES (40163635, '{"image":{"name":"$AWS_AMAZONLINUX_IMAGE"},"artifacts":{"paths":["${ASSET_NAME}/out"]},"before_script":["yum install -y python3 python3-pip zip findutils","cd ${ASSET_NAME}"],"script":["mkdir -p venv","cd venv","python3 -m venv pyspark_venvsource","source pyspark_venvsource/bin/activate","pip3 install -r ../scripts/requirements.txt","pip3 install venv-pack","venv-pack -f -o pyspark_venv.tar.gz","cd ..; mkdir -p out/lib","cd src/step/; cp -r --parents ./*/*main*.py ../../out"],"retry":{"max":2}}', '[{"key":"ASSET_NAME","value":"data-pipeline"}]', FALSE, 6269937106, 101) RETURNING "id")
+    }
+    test_parse(sql, {}, 'sql', true, expected)
+  end
+
   test 'normal comment prepended' do
     sql = '/* this is just a comment */ SELECT COUNT(*) FROM "projects"'
     expected = {
@@ -87,6 +100,19 @@ class Marginalia < Test::Unit::TestCase
     test_parse(sql, {}, 'sql', true, expected)
   end
 
+  test 'marginalia prepended with other comments in SQL' do
+    sql = %(/*application:sidekiq,correlation_id:01HQNQAHYZF4Q294P59R7RP71H,jid:047ecd6920dc27ccd49b0aef,endpoint_id:PostReceive,db_config_name:ci*/ INSERT INTO "p_ci_builds_metadata" ("project_id", "config_options", "config_variables", "has_exposed_artifacts", "build_id", "partition_id") VALUES (40163635, '{"image":{"name":"$AWS_AMAZONLINUX_IMAGE"},"artifacts":{"paths":["${ASSET_NAME}/out"]},"before_script":["yum install -y python3 python3-pip zip findutils","cd ${ASSET_NAME}"],"script":["mkdir -p venv","cd venv","python3 -m venv pyspark_venvsource","source pyspark_venvsource/bin/activate","pip3 install -r ../scripts/requirements.txt","pip3 install venv-pack","venv-pack -f -o pyspark_venv.tar.gz","cd ..; mkdir -p out/lib","cd src/step/; cp -r --parents ./*/*main*.py ../../out"],"retry":{"max":2}}', '[{"key":"ASSET_NAME","value":"data-pipeline"}]', FALSE, 6269937106, 101) RETURNING "id")
+    expected = {
+      'application' => 'sidekiq',
+      'correlation_id' => '01HQNQAHYZF4Q294P59R7RP71H',
+      'jid' => '047ecd6920dc27ccd49b0aef',
+      'db_config_name' => 'ci',
+      'endpoint_id' => 'PostReceive',
+      'sql' => %(INSERT INTO "p_ci_builds_metadata" ("project_id", "config_options", "config_variables", "has_exposed_artifacts", "build_id", "partition_id") VALUES (40163635, '{"image":{"name":"$AWS_AMAZONLINUX_IMAGE"},"artifacts":{"paths":["${ASSET_NAME}/out"]},"before_script":["yum install -y python3 python3-pip zip findutils","cd ${ASSET_NAME}"],"script":["mkdir -p venv","cd venv","python3 -m venv pyspark_venvsource","source pyspark_venvsource/bin/activate","pip3 install -r ../scripts/requirements.txt","pip3 install venv-pack","venv-pack -f -o pyspark_venv.tar.gz","cd ..; mkdir -p out/lib","cd src/step/; cp -r --parents ./*/*main*.py ../../out"],"retry":{"max":2}}', '[{"key":"ASSET_NAME","value":"data-pipeline"}]', FALSE, 6269937106, 101) RETURNING "id")
+    }
+    test_parse(sql, {}, 'sql', true, expected)
+  end
+
   test 'marginalia prepended for web, comment_strip disabled' do
     sql = 'SELECT COUNT(*) FROM "projects" /*application:sidekiq,correlation_id:d67cae54c169e0cab7d73389e2934f0e,jid:52a1c8a9e4c555ea573f20f0,job_class:Geo::MetricsUpdateWorker*/'
     expected = {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-postgresql-csvlog
 version: !ruby/object:Gem::Version
-  version: 0.8.0
+  version: 0.9.0
 platform: ruby
 authors:
 - stanhu
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-08-27 00:00:00.000000000 Z
+date: 2024-05-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -36,28 +36,28 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: '1.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.1'
+        version: '1.5'
 - !ruby/object:Gem::Dependency
   name: pg_query
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '5.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.0'
+        version: '5.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -140,18 +140,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.19
+rubygems_version: 3.3.27
 signing_key:
 specification_version: 4
 summary: fluentd plugins to work with PostgreSQL CSV logs
-test_files:
-- test/helper.rb
-- test/plugin/itest_in_pg_stat_activity.rb
-- test/plugin/itest_in_pg_stat_statements.rb
-- test/plugin/test_filter_marginalia.rb
-- test/plugin/test_filter_postgresql_redactor.rb
-- test/plugin/test_filter_postgresql_slowlog.rb
-- test/plugin/test_in_pg_stat_activity.rb
-- test/plugin/test_in_pg_stat_statements.rb
-- test/plugin/test_marginalia_extractor.rb
-- test/verify-docker-compose.sh
+test_files: []