fluent-plugin-postgresql-csvlog 0.7.3 → 0.8.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f0693e1f78345ee808b7ad9b58c76c6ab97863c6f3e5beb12207bf3bffae5761
-  data.tar.gz: 0ad321b2d4c797c110c690afd60018eb2cc3d37b6bf2f735ea475285bf0022b5
+  metadata.gz: d9f81f14bc67d4554142164c18775964dacf12feaa57ddc5d6a64b67944a9357
+  data.tar.gz: deb17cb65f08f9d94fc53720d47cafe67022ae6999c6765023bf6d87e39d0463
 SHA512:
-  metadata.gz: 582c67cf06d3a6c94378408e76712c130d91fdb7ae239deb272dbcaf97f7ba509d243dc45e3f525ecdb3da82271b0f94f1f2183a59b7fb09e054c66844fb0c63
-  data.tar.gz: 88ba2dc6cc7490a08389ca7d471f78f4069b5218e604d11064a83d76e2abc4216110f1012f0840e71b7682d5223fc3298c8cb8f3c29c3e6b9a400881bda24ef0
+  metadata.gz: 287762c56603630b1603ba523859e4114d3f25a2d9c39db6661201f0703f44eb4f361b78acb93b6897119c4e7e511725925f6567a3b938d9735d853a42467112
+  data.tar.gz: 8676d896c51ae02b943435b4fa0c607e340cc2572fae04f28d989aaa77eec5ec2558b2fc1f4f3bff6c0b97850c689f966a331a2ff5c459c2402a5b9b562b4bf1

data/.gitlab-ci.yml

@@ -1,16 +1,32 @@
-image: "ruby:2.7"
+image: "ruby:${RUBY_VERSION}"
+
+variables:
+  RUBY_VERSION: "3.2"
+
+stages:
+  - test
+  - deploy
+
+include:
+  - component: gitlab.com/gitlab-org/components/gem-release/gem-release@~latest
+    inputs:
+      smoke_test_script: "ruby -r 'fluent/plugin/filter_postgresql_slowlog'"
+      file_pattern_to_trigger_release: "fluent-plugin-postgresql-csvlog.gemspec"
 
 test:
+  stage: test
   before_script:
     - bundle config set path vendor
     - bundle install --jobs $(nproc)
   script:
     - bundle exec rake test
   cache:
+    key: ruby-${RUBY_VERSION}
     paths:
       - vendor/ruby
 
 .iteration_test:
+  stage: test
   services:
     - name: postgres:$POSTGRES_SERVER_VERSION
       alias: postgres
@@ -27,25 +43,22 @@ test:
     paths:
       - vendor/ruby
 
-# integration tests for postgres 12
-itest_pg12:
+# integration tests for PostgreSQL
+itest:
   extends: .iteration_test
-  variables:
-    POSTGRES_SERVER_VERSION: 12
-
-# integration tests for postgres 13
-itest_pg13:
-  extends: .iteration_test
-  variables:
-    POSTGRES_SERVER_VERSION: 13
+  parallel:
+    matrix:
+      - POSTGRES_SERVER_VERSION: ["12", "13", "14", "15", "16"]
 
 end_to_end_verification_test:
-  image: docker:19.03.12
+  stage: test
+  image: docker:20.10.16
   services:
-    - docker:19.03.12-dind
+    - docker:20.10.16-dind
   tags:
     - gitlab-org-docker
   variables:
+    DOCKER_HOST: tcp://docker:2375
     DOCKER_TLS_CERTDIR: ""
   before_script:
     - apk add --no-cache docker-compose

data/docker-compose.yml

@@ -2,7 +2,7 @@
 version: "3.3"
 services:
   postgres:
-    image: postgres:13
+    image: postgres:14
     restart: "no"
     environment:
       - POSTGRES_USER=testuser

data/fluent-plugin-postgresql-csvlog.gemspec

@@ -2,7 +2,7 @@ $:.push File.expand_path('lib', __dir__)
 
 Gem::Specification.new do |s|
   s.name        = 'fluent-plugin-postgresql-csvlog'
-  s.version     = '0.7.3'
+  s.version     = '0.8.2'
   s.authors     = ['stanhu']
   s.email       = ['stanhu@gmail.com']
   s.homepage    = 'https://gitlab.com/gitlab-org/fluent-plugins/fluent-plugin-postgresql-csvlog'

data/lib/fluent/plugin/in_pg_stat_activity.rb

@@ -28,11 +28,11 @@ module Fluent::Plugin
         wait_event_type,
         wait_event,
         xact_start,
-        extract(epoch from clock_timestamp() - xact_start) xact_age_s,
+        CAST(extract(epoch from clock_timestamp() - xact_start) AS double precision) xact_age_s,
         query_start,
-        extract(epoch from clock_timestamp() - query_start) query_age_s,
+        CAST(extract(epoch from clock_timestamp() - query_start) AS double precision) query_age_s,
         state_change,
-        extract(epoch from clock_timestamp() - state_change) state_age_s,
+        CAST(extract(epoch from clock_timestamp() - state_change) AS double precision) state_age_s,
         state,
         query
       FROM pg_catalog.pg_stat_activity

data/lib/fluent/plugin/marginalia_extractor.rb

@@ -6,8 +6,8 @@ module Fluent::Plugin
   # MarginaliaExtractor provides the parse_marginalia_into_record
   # utility method, useful for extracting marginalia into fluentd records
   module MarginaliaExtractor
-    MARGINALIA_PREPENDED_REGEXP = %r{^(?<comment>/\*.*\*/)(?<sql>.*)}m.freeze
-    MARGINALIA_APPENDED_REGEXP = %r{(?<sql>.*)(?<comment>/\*.*\*/)\s*;?\s*$}m.freeze
+    MARGINALIA_PREPENDED_REGEXP = %r{^(?<comment>/\*.*?\*/)\s*(?<sql>.*)}m
+    MARGINALIA_APPENDED_REGEXP = %r{(?<sql>.*)(?<comment>/\*.*\*/)\s*;?\s*$}m
 
     # Injects marginalia into a fluentd record
     def parse_marginalia_into_record(record, key, strip_comment)

data/test/plugin/itest_in_pg_stat_activity.rb

@@ -110,6 +110,11 @@ class PgStatActivityInputIntegrationTest < Test::Unit::TestCase
       assert_false record['query_length'].nil?
       assert_false record['query'].nil?
       assert_false record['fingerprint'].nil?
+
+      assert_true record['xact_age_s'].class == Float
+      assert_true record['query_age_s'].class == Float
+      assert_true record['state_age_s'].class == Float
+      assert_empty record.values.select { |val| val.class == BigDecimal }
     end
   end
 end

data/test/plugin/test_marginalia_extractor.rb

@@ -48,6 +48,19 @@ class Marginalia < Test::Unit::TestCase
     test_parse(sql, {}, 'sql', true, expected)
   end
 
+  test 'marginalia appended with other comments in SQL' do
+    sql = %(INSERT INTO "p_ci_builds_metadata" ("project_id", "config_options", "config_variables", "has_exposed_artifacts", "build_id", "partition_id") VALUES (40163635, '{"image":{"name":"$AWS_AMAZONLINUX_IMAGE"},"artifacts":{"paths":["${ASSET_NAME}/out"]},"before_script":["yum install -y python3 python3-pip zip findutils","cd ${ASSET_NAME}"],"script":["mkdir -p venv","cd venv","python3 -m venv pyspark_venvsource","source pyspark_venvsource/bin/activate","pip3 install -r ../scripts/requirements.txt","pip3 install venv-pack","venv-pack -f -o pyspark_venv.tar.gz","cd ..; mkdir -p out/lib","cd src/step/; cp -r --parents ./*/*main*.py ../../out"],"retry":{"max":2}}', '[{"key":"ASSET_NAME","value":"data-pipeline"}]', FALSE, 6269937106, 101) RETURNING "id" /*application:sidekiq,correlation_id:01HQNQAHYZF4Q294P59R7RP71H,jid:047ecd6920dc27ccd49b0aef,endpoint_id:PostReceive,db_config_name:ci*/)
+    expected = {
+      'application' => 'sidekiq',
+      'correlation_id' => '01HQNQAHYZF4Q294P59R7RP71H',
+      'jid' => '047ecd6920dc27ccd49b0aef',
+      'db_config_name' => 'ci',
+      'endpoint_id' => 'PostReceive',
+      'sql' => %(INSERT INTO "p_ci_builds_metadata" ("project_id", "config_options", "config_variables", "has_exposed_artifacts", "build_id", "partition_id") VALUES (40163635, '{"image":{"name":"$AWS_AMAZONLINUX_IMAGE"},"artifacts":{"paths":["${ASSET_NAME}/out"]},"before_script":["yum install -y python3 python3-pip zip findutils","cd ${ASSET_NAME}"],"script":["mkdir -p venv","cd venv","python3 -m venv pyspark_venvsource","source pyspark_venvsource/bin/activate","pip3 install -r ../scripts/requirements.txt","pip3 install venv-pack","venv-pack -f -o pyspark_venv.tar.gz","cd ..; mkdir -p out/lib","cd src/step/; cp -r --parents ./*/*main*.py ../../out"],"retry":{"max":2}}', '[{"key":"ASSET_NAME","value":"data-pipeline"}]', FALSE, 6269937106, 101) RETURNING "id")
+    }
+    test_parse(sql, {}, 'sql', true, expected)
+  end
+
   test 'normal comment prepended' do
     sql = '/* this is just a comment */ SELECT COUNT(*) FROM "projects"'
     expected = {
@@ -87,6 +100,19 @@ class Marginalia < Test::Unit::TestCase
     test_parse(sql, {}, 'sql', true, expected)
   end
 
+  test 'marginalia prepended with other comments in SQL' do
+    sql = %(/*application:sidekiq,correlation_id:01HQNQAHYZF4Q294P59R7RP71H,jid:047ecd6920dc27ccd49b0aef,endpoint_id:PostReceive,db_config_name:ci*/ INSERT INTO "p_ci_builds_metadata" ("project_id", "config_options", "config_variables", "has_exposed_artifacts", "build_id", "partition_id") VALUES (40163635, '{"image":{"name":"$AWS_AMAZONLINUX_IMAGE"},"artifacts":{"paths":["${ASSET_NAME}/out"]},"before_script":["yum install -y python3 python3-pip zip findutils","cd ${ASSET_NAME}"],"script":["mkdir -p venv","cd venv","python3 -m venv pyspark_venvsource","source pyspark_venvsource/bin/activate","pip3 install -r ../scripts/requirements.txt","pip3 install venv-pack","venv-pack -f -o pyspark_venv.tar.gz","cd ..; mkdir -p out/lib","cd src/step/; cp -r --parents ./*/*main*.py ../../out"],"retry":{"max":2}}', '[{"key":"ASSET_NAME","value":"data-pipeline"}]', FALSE, 6269937106, 101) RETURNING "id")
+    expected = {
+      'application' => 'sidekiq',
+      'correlation_id' => '01HQNQAHYZF4Q294P59R7RP71H',
+      'jid' => '047ecd6920dc27ccd49b0aef',
+      'db_config_name' => 'ci',
+      'endpoint_id' => 'PostReceive',
+      'sql' => %(INSERT INTO "p_ci_builds_metadata" ("project_id", "config_options", "config_variables", "has_exposed_artifacts", "build_id", "partition_id") VALUES (40163635, '{"image":{"name":"$AWS_AMAZONLINUX_IMAGE"},"artifacts":{"paths":["${ASSET_NAME}/out"]},"before_script":["yum install -y python3 python3-pip zip findutils","cd ${ASSET_NAME}"],"script":["mkdir -p venv","cd venv","python3 -m venv pyspark_venvsource","source pyspark_venvsource/bin/activate","pip3 install -r ../scripts/requirements.txt","pip3 install venv-pack","venv-pack -f -o pyspark_venv.tar.gz","cd ..; mkdir -p out/lib","cd src/step/; cp -r --parents ./*/*main*.py ../../out"],"retry":{"max":2}}', '[{"key":"ASSET_NAME","value":"data-pipeline"}]', FALSE, 6269937106, 101) RETURNING "id")
+    }
+    test_parse(sql, {}, 'sql', true, expected)
+  end
+
   test 'marginalia prepended for web, comment_strip disabled' do
     sql = 'SELECT COUNT(*) FROM "projects" /*application:sidekiq,correlation_id:d67cae54c169e0cab7d73389e2934f0e,jid:52a1c8a9e4c555ea573f20f0,job_class:Geo::MetricsUpdateWorker*/'
     expected = {

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-postgresql-csvlog
 version: !ruby/object:Gem::Version
-  version: 0.7.3
+  version: 0.8.2
 platform: ruby
 authors:
 - stanhu
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-04-11 00:00:00.000000000 Z
+date: 2024-02-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -140,7 +140,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.10
+rubygems_version: 3.5.6
 signing_key:
 specification_version: 4
 summary: fluentd plugins to work with PostgreSQL CSV logs