fluent-plugin-cloudfront-log-optimized 0.2.1 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (7) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +3 -0
  3. data/benchmark-gsub-vs-replace.rb +23 -0
  4. data/fluent-plugin-cloudfront-log-optimized.gemspec +1 -1
  5. data/lib/fluent/plugin/in_cloudfront_log.rb +4 -1
  6. data/test/plugin/test_in_cloudfrontlog.rb +16 -0
  7. metadata +4 -3
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: cbbdcfd3c3c07b1d9d8f8f6dd0a16d13a788e6b87210f91b33e48536e5cf5ee9
4
- data.tar.gz: 4418c4f338ca6bcd5718728ce79a6293263dece6e20d12472e65e0548601ad71
3
+ metadata.gz: 29422143da7c36ffd474f6f9345ca3419fc9eec9315e015edf99a8de4aa763ca
4
+ data.tar.gz: 22be1e3ad171f1a684b4eb11ed3fa4584345b1af8ceabddb721dc08edba22135
5
5
  SHA512:
6
- metadata.gz: 9e44f3a6d3a2cf65a45a2cf5c106bbbec9c62376cc03477541c168f82b649698e1f9480ae03410528fe351481b91bde9ebf42880ef07191a1bedf3197d34e8a7
7
- data.tar.gz: ce1791412ff7169171be180d79e350bddf336e06fa3801c9ed5138513d656ccdb06f0c621a589e445ae59fda1f6262d0cd53e7fe9309678981580ae8822f2cc3
6
+ metadata.gz: 2c5a78c69cda9394786b3e5bb7ce81e30d2f61a8b975e9c0b9d7d00564d42589842d59cd3ddc616921a0c280e0bea213b4fc7a953f55094897491f1eda878004
7
+ data.tar.gz: 6861b08d29b8144e9763f5ee235f9c0e493bd52061c8dff003e9e24e565cdfeea56b2236f691ec6bf1f6d40b5f11baa17abdaec8aff51809d929f4cb1a8173ff
data/CHANGELOG.md CHANGED
@@ -1,3 +1,6 @@
1
+ ## Release 0.2.2
2
+ - [end] replace slow gsub with more efficient string replace
3
+
1
4
  ## Release 0.2.1
2
5
  Merge pull request #1 from kdgm/feature/fix/escaped-tab-handling
3
6
 
data/benchmark-gsub-vs-replace.rb ADDED
@@ -0,0 +1,23 @@
1
+ require "benchmark"
2
+ require "cgi"
3
+
4
+ n = 5000000
5
+ log_line = '2025-10-12 09:52:59 MRS53-P3 1400 150.107.232.112 POST d2p1j3y3mcauy0.cloudfront.net /plugin/add 403 - Mozilla/5.0%20(Macintosh;%20Intel%20Mac%20OS%20X%2010_15_7)%20AppleWebKit/605.1.15%20(KHTML,%20like%20Gecko)%20Version/17.3.1%20Safari/605.1.1%0920.51 - - Error WcPCNG0WyL4BbXhEXq4AQulhrqte2TPUHt1Uz-iqcSwtx1L6ORdTOA== livecdn.kerkdienstgemist.nl https 9760 0.124 - TLSv1.3 TLS_AES_128_GCM_SHA256 Error HTTP/1.1 - - 50294 0.000 InvalidRequestMethod text/html 1053 - -'
6
+ Benchmark.bm do |x|
7
+ x.report do
8
+ n.times do
9
+ log_line = '2025-10-12 09:52:59 MRS53-P3 1400 150.107.232.112 POST d2p1j3y3mcauy0.cloudfront.net /plugin/add 403 - Mozilla/5.0%20(Macintosh;%20Intel%20Mac%20OS%20X%2010_15_7)%20AppleWebKit/605.1.15%20(KHTML,%20like%20Gecko)%20Version/17.3.1%20Safari/605.1.1%0920.51 - - Error WcPCNG0WyL4BbXhEXq4AQulhrqte2TPUHt1Uz-iqcSwtx1L6ORdTOA== livecdn.kerkdienstgemist.nl https 9760 0.124 - TLSv1.3 TLS_AES_128_GCM_SHA256 Error HTTP/1.1 - - 50294 0.000 InvalidRequestMethod text/html 1053 - -'
10
+ CGI.unescape(log_line.gsub('%09', " "))
11
+ log_line
12
+ end
13
+ end
14
+
15
+ x.report do
16
+ n.times do
17
+ log_line = '2025-10-12 09:52:59 MRS53-P3 1400 150.107.232.112 POST d2p1j3y3mcauy0.cloudfront.net /plugin/add 403 - Mozilla/5.0%20(Macintosh;%20Intel%20Mac%20OS%20X%2010_15_7)%20AppleWebKit/605.1.15%20(KHTML,%20like%20Gecko)%20Version/17.3.1%20Safari/605.1.1%0920.51 - - Error WcPCNG0WyL4BbXhEXq4AQulhrqte2TPUHt1Uz-iqcSwtx1L6ORdTOA== livecdn.kerkdienstgemist.nl https 9760 0.124 - TLSv1.3 TLS_AES_128_GCM_SHA256 Error HTTP/1.1 - - 50294 0.000 InvalidRequestMethod text/html 1053 - -'
18
+ log_line["%09"] = " " if log_line.include?("%09")
19
+ CGI.unescape(log_line)
20
+ log_line
21
+ end
22
+ end
23
+ end
data/fluent-plugin-cloudfront-log-optimized.gemspec CHANGED
@@ -4,7 +4,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
4
 
5
5
  Gem::Specification.new do |spec|
6
6
  spec.name = "fluent-plugin-cloudfront-log-optimized"
7
- spec.version = "0.2.1"
7
+ spec.version = "0.2.2"
8
8
  spec.authors = ["kubihee", "lenfree", "kjwierenga"]
9
9
  spec.email = ["kubihie@gmail.com", "lenfree.yeung@gmail.com", "k.j.wierenga@gmail.com"]
10
10
 
data/lib/fluent/plugin/in_cloudfront_log.rb CHANGED
@@ -132,9 +132,12 @@ class Fluent::Cloudfront_LogInput < Fluent::Input
132
132
  return
133
133
  end
134
134
 
135
+ # replace %09 (tab) with space to avoid incorrect introduction of tab character by CGI.unescape
136
+ line["%09"] = " " if line.include?("%09")
137
+
135
138
  record = [
136
139
  @fields,
137
- CGI.unescape(line.gsub('%09', ' ')).strip.split("\t") # don't unescape tab (%09)
140
+ CGI.unescape(line).split("\t")
138
141
  ].transpose.to_h
139
142
 
140
143
  timestamp = if @parse_date_time
data/test/plugin/test_in_cloudfrontlog.rb CHANGED
@@ -121,6 +121,22 @@ class Cloudfront_LogInputTest < Test::Unit::TestCase
121
121
 
122
122
  assert_equal(emitted_event['cs(User-Agent)'], "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.3.1 Safari/605.1.1 20.51")
123
123
  end
124
+
125
+ test "log line without %09 in user-agent is parsed correctly" do
126
+ driver = create_driver(MINIMAL_CONFIG)
127
+ instance = driver.instance
128
+
129
+ version_line = "#Version: 1.0"
130
+ fields_line = "#Fields: date time x-edge-location sc-bytes c-ip cs-method cs(Host) cs-uri-stem sc-status cs(Referer) cs(User-Agent) cs-uri-query cs(Cookie) x-edge-result-type x-edge-request-id x-host-header cs-protocol cs-bytes time-taken x-forwarded-for ssl-protocol ssl-cipher x-edge-response-result-type cs-protocol-version fle-status fle-encrypted-fields c-port time-to-first-byte x-edge-detailed-result-type sc-content-type sc-content-len sc-range-start sc-range-end"
131
+ regression_line_with_excaped_tab = "2025-10-12 09:52:59 MRS53-P3 1400 150.107.232.112 POST d2p1j3y3mcauy0.cloudfront.net /plugin/add 403 - Mozilla/5.0%20(Macintosh;%20Intel%20Mac%20OS%20X%2010_15_7)%20AppleWebKit/605.1.15%20(KHTML,%20like%20Gecko)%20Version/17.3.1%20Safari/605.1.1-20.51 - - Error WcPCNG0WyL4BbXhEXq4AQulhrqte2TPUHt1Uz-iqcSwtx1L6ORdTOA== livecdn.kerkdienstgemist.nl https 9760 0.124 - TLSv1.3 TLS_AES_128_GCM_SHA256 Error HTTP/1.1 - - 50294 0.000 InvalidRequestMethod text/html 1053 - -"
132
+
133
+ # Prime the processor with version and fields lines
134
+ instance.process_line(version_line)
135
+ instance.process_line(fields_line)
136
+ emitted_event = instance.process_line(regression_line_with_excaped_tab)
137
+
138
+ assert_equal(emitted_event['cs(User-Agent)'], "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.3.1 Safari/605.1.1-20.51")
139
+ end
124
140
  end
125
141
 
126
142
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: fluent-plugin-cloudfront-log-optimized
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.2.1
4
+ version: 0.2.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - kubihee
@@ -10,7 +10,7 @@ authors:
10
10
  autorequire:
11
11
  bindir: bin
12
12
  cert_chain: []
13
- date: 2025-10-24 00:00:00.000000000 Z
13
+ date: 2025-11-10 00:00:00.000000000 Z
14
14
  dependencies:
15
15
  - !ruby/object:Gem::Dependency
16
16
  name: fluentd
@@ -118,6 +118,7 @@ files:
118
118
  - Gemfile
119
119
  - README.md
120
120
  - Rakefile
121
+ - benchmark-gsub-vs-replace.rb
121
122
  - fluent-plugin-cloudfront-log-optimized.gemspec
122
123
  - lib/fluent/plugin/enumerable_inflater.rb
123
124
  - lib/fluent/plugin/in_cloudfront_log.rb
@@ -141,7 +142,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
141
142
  - !ruby/object:Gem::Version
142
143
  version: '0'
143
144
  requirements: []
144
- rubygems_version: 3.1.6
145
+ rubygems_version: 3.2.3
145
146
  signing_key:
146
147
  specification_version: 4
147
148
  summary: AWS CloudFront log input plugin optimized for large log files. Credit to