fluent-plugin-cloudfront-log-optimized 0.2.0 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 41c311f36c1f68872f1ad935ea398c3f21b0eb2154ed2ded25934f2e70966ab6
-  data.tar.gz: 0a6997034d17b4b5c02c66ac3dd8017cc842b40b3025bdcbb18c68719641536c
+  metadata.gz: 29422143da7c36ffd474f6f9345ca3419fc9eec9315e015edf99a8de4aa763ca
+  data.tar.gz: 22be1e3ad171f1a684b4eb11ed3fa4584345b1af8ceabddb721dc08edba22135
 SHA512:
-  metadata.gz: 87aec8f7754aefe0a81b47a7beb255d6b1598213a73623117cad56206e5f7b263ac3118963db64e65b1906f947ab448fe2d3e70448d14a74f5aeb6e155b8fd13
-  data.tar.gz: f77b92a998259fbf4b3c778b4032c87e3860b37355e10631cc01dce3c099f89bf094cab113c7e9f62875d8701b1910c17b289236addd726245eed3b6c7c7c905
+  metadata.gz: 2c5a78c69cda9394786b3e5bb7ce81e30d2f61a8b975e9c0b9d7d00564d42589842d59cd3ddc616921a0c280e0bea213b4fc7a953f55094897491f1eda878004
+  data.tar.gz: 6861b08d29b8144e9763f5ee235f9c0e493bd52061c8dff003e9e24e565cdfeea56b2236f691ec6bf1f6d40b5f11baa17abdaec8aff51809d929f4cb1a8173ff

data/.devcontainer/Dockerfile

@@ -0,0 +1,42 @@
+ARG RUBY_VERSION=2.7.6
+ARG RUBYGEMS_UPDATE_VERSION=1.8.30
+ARG BUNDLER_VERSION=1.7.15
+
+FROM ruby:$RUBY_VERSION-slim
+LABEL maintainer="k.j.wierenga@kerkdienstgemist.nl"
+
+# Common dependencies
+# Install apt based dependencies required to run Rails as
+# well as RubyGems. As the Ruby image itself is based on a
+# Debian image, we use apt-get to install those.
+RUN apt-get update -qq \
+    && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+        build-essential \
+        git \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
+    && truncate -s 0 /var/log/*log
+
+# Configure bundler
+ENV LANG=C.UTF-8 \
+    BUNDLE_JOBS=4 \
+    BUNDLE_RETRY=3
+
+# Uncomment this line if you store Bundler settings in the project's root
+# ENV BUNDLE_APP_CONFIG=.bundle
+
+# Uncomment this line if you want to run binstubs without prefixing with `bin/` or `bundle exec`
+# ENV PATH /app/bin:$PATH
+
+# Upgrade RubyGems and install required Bundler version
+# ARG BUNDLER_VERSION
+# ARG RUBYGEMS_UPDATE_VERSION
+# RUN gem uninstall bundler && \
+#     gem update --system $RUBYGEMS_UPDATE_VERSION && \
+#     gem install bundler:$BUNDLER_VERSION
+
+# Configure the main working directory. This is the base
+# directory used in any further RUN, COPY, and ENTRYPOINT commands.
+ENV WORKDIR=/app
+RUN mkdir -p $WORKDIR
+WORKDIR $WORKDIR

data/.devcontainer/devcontainer.json

@@ -0,0 +1,21 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the
+// README at: https://github.com/devcontainers/templates/tree/main/src/ruby
+{
+	"name": "Ruby",
+	// Or use a Dockerfile or Docker Compose file. More info: https://containers.dev/guide/dockerfile
+	// "image": "ruby:2.7.6-slim"
+	"build": {
+		// Path is relative to the devcontainer.json file.
+		"dockerfile": "Dockerfile"
+	}
+	// Features to add to the dev container. More info: https://containers.dev/features.
+	// "features": {},
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "ruby --version",
+	// Configure tool-specific properties.
+	// "customizations": {},
+	// Uncomment to connect as root instead. More info: https://aka.ms/dev-containers-non-root.
+	// "remoteUser": "root"
+}

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## Release 0.2.2
+- [end] replace slow gsub with more efficient string replace
+
+## Release 0.2.1
+Merge pull request #1 from kdgm/feature/fix/escaped-tab-handling
+
 ## Release 0.2.0 (kjwierenga)
 Merge branch 'feature/enh/rename-to-optimized' into develop
 Merge branch 'feature/fix/process-log-files-in-last-modified-order' into develop

data/benchmark-gsub-vs-replace.rb

@@ -0,0 +1,23 @@
+require "benchmark"
+require "cgi"
+
+n = 5000000
+log_line = '2025-10-12	09:52:59	MRS53-P3	1400	150.107.232.112	POST	d2p1j3y3mcauy0.cloudfront.net	/plugin/add	403	-	Mozilla/5.0%20(Macintosh;%20Intel%20Mac%20OS%20X%2010_15_7)%20AppleWebKit/605.1.15%20(KHTML,%20like%20Gecko)%20Version/17.3.1%20Safari/605.1.1%0920.51	-	-	Error	WcPCNG0WyL4BbXhEXq4AQulhrqte2TPUHt1Uz-iqcSwtx1L6ORdTOA==	livecdn.kerkdienstgemist.nl	https	9760	0.124	-	TLSv1.3	TLS_AES_128_GCM_SHA256	Error	HTTP/1.1	-	-	50294	0.000	InvalidRequestMethod	text/html	1053	-	-'
+Benchmark.bm do |x|
+  x.report do
+    n.times do
+      log_line = '2025-10-12	09:52:59	MRS53-P3	1400	150.107.232.112	POST	d2p1j3y3mcauy0.cloudfront.net	/plugin/add	403	-	Mozilla/5.0%20(Macintosh;%20Intel%20Mac%20OS%20X%2010_15_7)%20AppleWebKit/605.1.15%20(KHTML,%20like%20Gecko)%20Version/17.3.1%20Safari/605.1.1%0920.51	-	-	Error	WcPCNG0WyL4BbXhEXq4AQulhrqte2TPUHt1Uz-iqcSwtx1L6ORdTOA==	livecdn.kerkdienstgemist.nl	https	9760	0.124	-	TLSv1.3	TLS_AES_128_GCM_SHA256	Error	HTTP/1.1	-	-	50294	0.000	InvalidRequestMethod	text/html	1053	-	-'
+      CGI.unescape(log_line.gsub('%09', " "))
+      log_line
+    end
+  end
+
+  x.report do
+    n.times do
+      log_line = '2025-10-12	09:52:59	MRS53-P3	1400	150.107.232.112	POST	d2p1j3y3mcauy0.cloudfront.net	/plugin/add	403	-	Mozilla/5.0%20(Macintosh;%20Intel%20Mac%20OS%20X%2010_15_7)%20AppleWebKit/605.1.15%20(KHTML,%20like%20Gecko)%20Version/17.3.1%20Safari/605.1.1%0920.51	-	-	Error	WcPCNG0WyL4BbXhEXq4AQulhrqte2TPUHt1Uz-iqcSwtx1L6ORdTOA==	livecdn.kerkdienstgemist.nl	https	9760	0.124	-	TLSv1.3	TLS_AES_128_GCM_SHA256	Error	HTTP/1.1	-	-	50294	0.000	InvalidRequestMethod	text/html	1053	-	-'
+      log_line["%09"] = " " if log_line.include?("%09")
+      CGI.unescape(log_line)
+      log_line
+    end
+  end
+end

data/fluent-plugin-cloudfront-log-optimized.gemspec

@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name          = "fluent-plugin-cloudfront-log-optimized"
-  spec.version       = "0.2.0"
+  spec.version       = "0.2.2"
   spec.authors       = ["kubihee", "lenfree", "kjwierenga"]
   spec.email         = ["kubihie@gmail.com", "lenfree.yeung@gmail.com", "k.j.wierenga@gmail.com"]
 
@@ -20,7 +20,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "fluentd", ">= 0.14.0", "< 2"
   spec.add_dependency "aws-sdk-s3", "~> 1"
   spec.add_dependency "aws-sdk-sqs", "~> 1"
-  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "bundler", "~> 2.0"
   spec.add_development_dependency "rake", "~> 12"
   spec.add_development_dependency 'test-unit', "~> 2"
 end

data/lib/fluent/plugin/in_cloudfront_log.rb

@@ -132,18 +132,23 @@ class Fluent::Cloudfront_LogInput < Fluent::Input
       return
     end
 
+    # replace %09 (tab) with space to avoid incorrect introduction of tab character by CGI.unescape
+    line["%09"] = " " if line.include?("%09")
+
     record = [
       @fields,
-      CGI.unescape(line).strip.split("\t") # hoge%2520fuga -> hoge%20fuga
+      CGI.unescape(line).split("\t")
     ].transpose.to_h
 
     timestamp = if @parse_date_time
-                  Time.iso8601("#{record['date']}T#{record['time']}+00:00").to_i
-                else
-                  Time.now.to_i
-                end
+      Time.iso8601("#{record['date']}T#{record['time']}+00:00").to_i
+    else
+      Time.now.to_i
+    end
 
     router.emit(@tag, timestamp, record)
+
+    record # for testing purposes
   end
 
   def process_content(content)

data/test/plugin/test_in_cloudfrontlog.rb

@@ -105,4 +105,38 @@ class Cloudfront_LogInputTest < Test::Unit::TestCase
     end
   end
 
+  sub_test_case "regression test for %09 (tab) in log lines" do
+    test "log line with %09 in user-agent is parsed correctly" do
+      driver = create_driver(MINIMAL_CONFIG)
+      instance = driver.instance
+
+      version_line = "#Version: 1.0"
+      fields_line = "#Fields: date time x-edge-location sc-bytes c-ip cs-method cs(Host) cs-uri-stem sc-status cs(Referer) cs(User-Agent) cs-uri-query cs(Cookie) x-edge-result-type x-edge-request-id x-host-header cs-protocol cs-bytes time-taken x-forwarded-for ssl-protocol ssl-cipher x-edge-response-result-type cs-protocol-version fle-status fle-encrypted-fields c-port time-to-first-byte x-edge-detailed-result-type sc-content-type sc-content-len sc-range-start sc-range-end"
+      regression_line_with_excaped_tab = "2025-10-12	09:52:59	MRS53-P3	1400	150.107.232.112	POST	d2p1j3y3mcauy0.cloudfront.net	/plugin/add	403	-	Mozilla/5.0%20(Macintosh;%20Intel%20Mac%20OS%20X%2010_15_7)%20AppleWebKit/605.1.15%20(KHTML,%20like%20Gecko)%20Version/17.3.1%20Safari/605.1.1%0920.51	-	-	Error	WcPCNG0WyL4BbXhEXq4AQulhrqte2TPUHt1Uz-iqcSwtx1L6ORdTOA==	livecdn.kerkdienstgemist.nl	https	9760	0.124	-	TLSv1.3	TLS_AES_128_GCM_SHA256	Error	HTTP/1.1	-	-	50294	0.000	InvalidRequestMethod	text/html	1053	-	-"
+
+      # Prime the processor with version and fields lines
+      instance.process_line(version_line)
+      instance.process_line(fields_line)
+      emitted_event = instance.process_line(regression_line_with_excaped_tab)
+
+      assert_equal(emitted_event['cs(User-Agent)'], "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.3.1 Safari/605.1.1 20.51")
+    end
+
+    test "log line without %09 in user-agent is parsed correctly" do
+      driver = create_driver(MINIMAL_CONFIG)
+      instance = driver.instance
+
+      version_line = "#Version: 1.0"
+      fields_line = "#Fields: date time x-edge-location sc-bytes c-ip cs-method cs(Host) cs-uri-stem sc-status cs(Referer) cs(User-Agent) cs-uri-query cs(Cookie) x-edge-result-type x-edge-request-id x-host-header cs-protocol cs-bytes time-taken x-forwarded-for ssl-protocol ssl-cipher x-edge-response-result-type cs-protocol-version fle-status fle-encrypted-fields c-port time-to-first-byte x-edge-detailed-result-type sc-content-type sc-content-len sc-range-start sc-range-end"
+      regression_line_with_excaped_tab = "2025-10-12	09:52:59	MRS53-P3	1400	150.107.232.112	POST	d2p1j3y3mcauy0.cloudfront.net	/plugin/add	403	-	Mozilla/5.0%20(Macintosh;%20Intel%20Mac%20OS%20X%2010_15_7)%20AppleWebKit/605.1.15%20(KHTML,%20like%20Gecko)%20Version/17.3.1%20Safari/605.1.1-20.51	-	-	Error	WcPCNG0WyL4BbXhEXq4AQulhrqte2TPUHt1Uz-iqcSwtx1L6ORdTOA==	livecdn.kerkdienstgemist.nl	https	9760	0.124	-	TLSv1.3	TLS_AES_128_GCM_SHA256	Error	HTTP/1.1	-	-	50294	0.000	InvalidRequestMethod	text/html	1053	-	-"
+
+      # Prime the processor with version and fields lines
+      instance.process_line(version_line)
+      instance.process_line(fields_line)
+      emitted_event = instance.process_line(regression_line_with_excaped_tab)
+
+      assert_equal(emitted_event['cs(User-Agent)'], "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.3.1 Safari/605.1.1-20.51")
+    end
+  end
+
 end

metadata

@@ -1,16 +1,16 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-cloudfront-log-optimized
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.2
 platform: ruby
 authors:
 - kubihee
 - lenfree
 - kjwierenga
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-02-06 00:00:00.000000000 Z
+date: 2025-11-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
@@ -66,14 +66,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '2.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.7'
+        version: '2.0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -111,11 +111,14 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".devcontainer/Dockerfile"
+- ".devcontainer/devcontainer.json"
 - ".gitignore"
 - CHANGELOG.md
 - Gemfile
 - README.md
 - Rakefile
+- benchmark-gsub-vs-replace.rb
 - fluent-plugin-cloudfront-log-optimized.gemspec
 - lib/fluent/plugin/enumerable_inflater.rb
 - lib/fluent/plugin/in_cloudfront_log.rb
@@ -124,7 +127,7 @@ files:
 homepage: https://github.com/kjwierenga/fluent-plugin-cloudfront-log-optimized
 licenses: []
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -139,8 +142,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.8
-signing_key:
+rubygems_version: 3.2.3
+signing_key: 
 specification_version: 4
 summary: AWS CloudFront log input plugin optimized for large log files. Credit to
   kubihie and lenfree.