fluentd-json-size-limit 0.1.4 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +24 -0
- data/LICENSE +16 -202
- data/NOTICE.md +3 -0
- data/README.md +61 -26
- data/fluentd-json-size-limit.gemspec +37 -20
- data/lib/fluent/plugin/filter_jsonsizelimit.rb +4 -67
- data/lib/fluent/plugin/json_size_limit/errors.rb +53 -0
- data/lib/fluent/plugin/json_size_limit/filter.rb +154 -0
- data/lib/fluent/plugin/json_size_limit/filter_configuration.rb +48 -0
- data/lib/fluent/plugin/json_size_limit/json_string.rb +169 -0
- data/lib/fluent/plugin/json_size_limit/processing_guard.rb +77 -0
- data/lib/fluent/plugin/json_size_limit/rate_limited_logger.rb +76 -0
- data/lib/fluent/plugin/json_size_limit/record_measurer.rb +133 -0
- data/lib/fluent/plugin/json_size_limit/record_processor.rb +52 -0
- data/lib/fluent/plugin/json_size_limit/record_reducer.rb +258 -0
- data/lib/fluent/plugin/json_size_limit/string_encoding.rb +25 -0
- data/lib/fluent/plugin/json_size_limit/telemetry.rb +60 -0
- data/lib/fluent/plugin/json_size_limit/version.rb +9 -0
- data/lib/fluent/plugin/json_size_limit.rb +15 -0
- metadata +143 -38
- data/.gitignore +0 -2
- data/.gitlab-ci.yml +0 -171
- data/Gemfile +0 -3
- data/Gemfile.lock +0 -51
- data/Rakefile +0 -13
- data/test/helper.rb +0 -7
- data/test/plugin/test_fluentd-json-size-limit.rb +0 -43
data/.gitlab-ci.yml
DELETED
|
@@ -1,171 +0,0 @@
|
|
|
1
|
-
default:
|
|
2
|
-
tags:
|
|
3
|
-
- k8s-tools-live
|
|
4
|
-
|
|
5
|
-
stages:
|
|
6
|
-
- start
|
|
7
|
-
- validate
|
|
8
|
-
- build
|
|
9
|
-
- publish
|
|
10
|
-
- final
|
|
11
|
-
|
|
12
|
-
notify_on_start:
|
|
13
|
-
variables:
|
|
14
|
-
GIT_STRATEGY: none
|
|
15
|
-
stage: start
|
|
16
|
-
script:
|
|
17
|
-
- |
|
|
18
|
-
if [ ! -z "${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}" ]; then
|
|
19
|
-
slack_send "Starting pipeline for MR of *$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME* branch into the *$CI_MERGE_REQUEST_TARGET_BRANCH_NAME* branch of the *$CI_PROJECT_NAME* docker image."
|
|
20
|
-
elif [ ! -z "${CI_COMMIT_TAG}" ]; then
|
|
21
|
-
slack_send "Starting pipeline for tag *$CI_COMMIT_TAG* of the *$CI_PROJECT_NAME* docker image."
|
|
22
|
-
elif [ ! -z "${CI_COMMIT_BRANCH}" ]; then
|
|
23
|
-
slack_send "Starting pipeline for branch *$CI_COMMIT_BRANCH* of the *$CI_PROJECT_NAME* docker image."
|
|
24
|
-
else
|
|
25
|
-
echo "Something went wront. Exiting..."
|
|
26
|
-
exit 1
|
|
27
|
-
fi
|
|
28
|
-
artifacts:
|
|
29
|
-
name: $CI_PIPELINE_ID
|
|
30
|
-
paths:
|
|
31
|
-
- slack_thread_ts
|
|
32
|
-
expire_in: 60min
|
|
33
|
-
when: always
|
|
34
|
-
rules:
|
|
35
|
-
- if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master"
|
|
36
|
-
when: always
|
|
37
|
-
changes:
|
|
38
|
-
- "*.gemsec"
|
|
39
|
-
- "*.yaml"
|
|
40
|
-
- "lib/fluent/plugin/*.rb"
|
|
41
|
-
- "test/plugin/*.rb"
|
|
42
|
-
- if: $CI_COMMIT_TAG =~ /^v\d{1,}\.\d{1,}\.\d{1,}$/ || $CI_COMMIT_BRANCH =~ /.*(\w{2,}-\d{1,}).*/
|
|
43
|
-
when: always
|
|
44
|
-
changes:
|
|
45
|
-
- "*.gemsec"
|
|
46
|
-
- "*.yaml"
|
|
47
|
-
- "lib/fluent/plugin/*.rb"
|
|
48
|
-
- "test/plugin/*.rb"
|
|
49
|
-
- when: never
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
validating:
|
|
53
|
-
stage: validate
|
|
54
|
-
script:
|
|
55
|
-
- bundle install
|
|
56
|
-
- bundle exec rake test
|
|
57
|
-
rules:
|
|
58
|
-
- if: '$CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master" || $CI_COMMIT_BRANCH =~ /.*(\w{2,}-\d{1,}).*/'
|
|
59
|
-
when: always
|
|
60
|
-
changes:
|
|
61
|
-
- "*.gemsec"
|
|
62
|
-
- "*.yaml"
|
|
63
|
-
- "lib/fluent/plugin/*.rb"
|
|
64
|
-
- "test/plugin/*.rb"
|
|
65
|
-
- when: never
|
|
66
|
-
|
|
67
|
-
build:
|
|
68
|
-
stage: build
|
|
69
|
-
script:
|
|
70
|
-
- gem build fluentd-json-size-limit.gemspec
|
|
71
|
-
artifacts:
|
|
72
|
-
paths:
|
|
73
|
-
- fluentd-json-size-limit-*.gem
|
|
74
|
-
expire_in: 60min
|
|
75
|
-
rules:
|
|
76
|
-
- if: '$CI_COMMIT_TAG'
|
|
77
|
-
when: on_success
|
|
78
|
-
changes:
|
|
79
|
-
- "*.gemsec"
|
|
80
|
-
- "*.yaml"
|
|
81
|
-
- "lib/fluent/plugin/*.rb"
|
|
82
|
-
- "test/plugin/*.rb"
|
|
83
|
-
|
|
84
|
-
publish:
|
|
85
|
-
stage: publish
|
|
86
|
-
before_script:
|
|
87
|
-
- AWS_CREDENTIALS=$(aws sts assume-role --role-arn arn:aws:iam::315805068186:role/SMR=rubygems-cicd-ci --role-session-name gitlab-secretsmanager-access)
|
|
88
|
-
- export AWS_ACCESS_KEY_ID=$(echo $AWS_CREDENTIALS | jq .Credentials | jq -r .AccessKeyId)
|
|
89
|
-
- export AWS_SECRET_ACCESS_KEY=$(echo $AWS_CREDENTIALS | jq .Credentials | jq -r .SecretAccessKey)
|
|
90
|
-
- export AWS_SESSION_TOKEN=$(echo $AWS_CREDENTIALS | jq .Credentials | jq -r .SessionToken)
|
|
91
|
-
- export GEM_HOST_API_KEY_SECRET=$(aws secretsmanager get-secret-value --secret-id rubygems/cicd/ci --region us-west-1)
|
|
92
|
-
- export GEM_HOST_API_KEY=$(echo $GEM_HOST_API_KEY_SECRET | jq .SecretString | jq fromjson | jq -r .GEM_HOST_API_KEY)
|
|
93
|
-
- unset AWS_ACCESS_KEY_ID
|
|
94
|
-
- unset AWS_SECRET_ACCESS_KEY
|
|
95
|
-
- unset AWS_SESSION_TOKEN
|
|
96
|
-
script:
|
|
97
|
-
- gem push fluentd-json-size-limit-*.gem
|
|
98
|
-
rules:
|
|
99
|
-
- if: '$CI_COMMIT_TAG'
|
|
100
|
-
when: on_success
|
|
101
|
-
changes:
|
|
102
|
-
- "*.gemsec"
|
|
103
|
-
- "*.yaml"
|
|
104
|
-
- "lib/fluent/plugin/*.rb"
|
|
105
|
-
- "test/plugin/*.rb"
|
|
106
|
-
|
|
107
|
-
notify_on_success:
|
|
108
|
-
variables:
|
|
109
|
-
GIT_STRATEGY: none
|
|
110
|
-
stage: final
|
|
111
|
-
dependencies:
|
|
112
|
-
- notify_on_start
|
|
113
|
-
script:
|
|
114
|
-
- |
|
|
115
|
-
if [ ! -z "${CI_COMMIT_TAG}" ]; then
|
|
116
|
-
slack_send "Pipeline for tag *$CI_COMMIT_TAG* of the *$CI_PROJECT_NAME* docker image succeeded" --status success
|
|
117
|
-
elif [ ! -z "${CI_COMMIT_BRANCH}" ] || [ ! -z "${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}" ]; then
|
|
118
|
-
slack_send "Pipeline for branch *$CI_COMMIT_BRANCH$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME* of the *$CI_PROJECT_NAME* docker image succeeded" --status success
|
|
119
|
-
else
|
|
120
|
-
echo "Something went wront. Exiting..."
|
|
121
|
-
exit 1
|
|
122
|
-
fi
|
|
123
|
-
rules:
|
|
124
|
-
- if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master"
|
|
125
|
-
when: on_success
|
|
126
|
-
changes:
|
|
127
|
-
- "*.gemsec"
|
|
128
|
-
- "*.yaml"
|
|
129
|
-
- "lib/fluent/plugin/*.rb"
|
|
130
|
-
- "test/plugin/*.rb"
|
|
131
|
-
- if: $CI_COMMIT_TAG =~ /^v\d{1,}\.\d{1,}\.\d{1,}$/ || $CI_COMMIT_BRANCH =~ /.*(\w{2,}-\d{1,}).*/
|
|
132
|
-
when: on_success
|
|
133
|
-
changes:
|
|
134
|
-
- "*.gemsec"
|
|
135
|
-
- "*.yaml"
|
|
136
|
-
- "lib/fluent/plugin/*.rb"
|
|
137
|
-
- "test/plugin/*.rb"
|
|
138
|
-
- when: never
|
|
139
|
-
|
|
140
|
-
notify_on_fail:
|
|
141
|
-
variables:
|
|
142
|
-
GIT_STRATEGY: none
|
|
143
|
-
stage: final
|
|
144
|
-
dependencies:
|
|
145
|
-
- notify_on_start
|
|
146
|
-
script:
|
|
147
|
-
- |
|
|
148
|
-
if [ ! -z "${CI_COMMIT_TAG}" ]; then
|
|
149
|
-
slack_send "Pipeline for tag *$CI_COMMIT_TAG* of the *$CI_PROJECT_NAME* docker image failed" --status fail
|
|
150
|
-
elif [ ! -z "${CI_COMMIT_BRANCH}" ] || [ ! -z "${CI_MERGE_REQUEST_TARGET_BRANCH_NAME}" ]; then
|
|
151
|
-
slack_send "Pipeline for branch *$CI_COMMIT_BRANCH$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME* of the *$CI_PROJECT_NAME* docker image failed" --status fail
|
|
152
|
-
else
|
|
153
|
-
echo "Something went wront. Exiting..."
|
|
154
|
-
exit 1
|
|
155
|
-
fi
|
|
156
|
-
rules:
|
|
157
|
-
- if: $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == "master"
|
|
158
|
-
when: on_failure
|
|
159
|
-
changes:
|
|
160
|
-
- "*.gemsec"
|
|
161
|
-
- "*.yaml"
|
|
162
|
-
- "lib/fluent/plugin/*.rb"
|
|
163
|
-
- "test/plugin/*.rb"
|
|
164
|
-
- if: $CI_COMMIT_TAG =~ /^v\d{1,}\.\d{1,}\.\d{1,}$/ || $CI_COMMIT_BRANCH =~ /.*(\w{2,}-\d{1,}).*/
|
|
165
|
-
when: on_failure
|
|
166
|
-
changes:
|
|
167
|
-
- "*.gemsec"
|
|
168
|
-
- "*.yaml"
|
|
169
|
-
- "lib/fluent/plugin/*.rb"
|
|
170
|
-
- "test/plugin/*.rb"
|
|
171
|
-
- when: never
|
data/Gemfile
DELETED
data/Gemfile.lock
DELETED
|
@@ -1,51 +0,0 @@
|
|
|
1
|
-
PATH
|
|
2
|
-
remote: .
|
|
3
|
-
specs:
|
|
4
|
-
fluentd-json-size-limit (0.1.1)
|
|
5
|
-
fluentd (>= 0.14.10, < 2)
|
|
6
|
-
|
|
7
|
-
GEM
|
|
8
|
-
remote: https://rubygems.org/
|
|
9
|
-
specs:
|
|
10
|
-
concurrent-ruby (1.2.2)
|
|
11
|
-
cool.io (1.8.0)
|
|
12
|
-
fluentd (1.16.3)
|
|
13
|
-
bundler
|
|
14
|
-
cool.io (>= 1.4.5, < 2.0.0)
|
|
15
|
-
http_parser.rb (>= 0.5.1, < 0.9.0)
|
|
16
|
-
msgpack (>= 1.3.1, < 2.0.0)
|
|
17
|
-
serverengine (>= 2.3.2, < 3.0.0)
|
|
18
|
-
sigdump (~> 0.2.5)
|
|
19
|
-
strptime (>= 0.2.4, < 1.0.0)
|
|
20
|
-
tzinfo (>= 1.0, < 3.0)
|
|
21
|
-
tzinfo-data (~> 1.0)
|
|
22
|
-
webrick (~> 1.4)
|
|
23
|
-
yajl-ruby (~> 1.0)
|
|
24
|
-
http_parser.rb (0.8.0)
|
|
25
|
-
msgpack (1.7.2)
|
|
26
|
-
power_assert (2.0.3)
|
|
27
|
-
rake (13.0.6)
|
|
28
|
-
serverengine (2.3.2)
|
|
29
|
-
sigdump (~> 0.2.2)
|
|
30
|
-
sigdump (0.2.5)
|
|
31
|
-
strptime (0.2.5)
|
|
32
|
-
test-unit (3.5.7)
|
|
33
|
-
power_assert
|
|
34
|
-
tzinfo (2.0.6)
|
|
35
|
-
concurrent-ruby (~> 1.0)
|
|
36
|
-
tzinfo-data (1.2023.3)
|
|
37
|
-
tzinfo (>= 1.0.0)
|
|
38
|
-
webrick (1.8.1)
|
|
39
|
-
yajl-ruby (1.4.3)
|
|
40
|
-
|
|
41
|
-
PLATFORMS
|
|
42
|
-
arm64-darwin-23
|
|
43
|
-
|
|
44
|
-
DEPENDENCIES
|
|
45
|
-
bundler (~> 2.4.22)
|
|
46
|
-
fluentd-json-size-limit!
|
|
47
|
-
rake (~> 13.0.6)
|
|
48
|
-
test-unit (~> 3.5.7)
|
|
49
|
-
|
|
50
|
-
BUNDLED WITH
|
|
51
|
-
2.4.22
|
data/Rakefile
DELETED
|
@@ -1,13 +0,0 @@
|
|
|
1
|
-
require "bundler"
|
|
2
|
-
Bundler::GemHelper.install_tasks
|
|
3
|
-
|
|
4
|
-
require "rake/testtask"
|
|
5
|
-
|
|
6
|
-
Rake::TestTask.new(:test) do |t|
|
|
7
|
-
t.libs.push("lib", "test")
|
|
8
|
-
t.test_files = FileList["test/**/test_*.rb"]
|
|
9
|
-
t.verbose = true
|
|
10
|
-
t.warning = true
|
|
11
|
-
end
|
|
12
|
-
|
|
13
|
-
task default: [:test]
|
data/test/helper.rb
DELETED
|
@@ -1,43 +0,0 @@
|
|
|
1
|
-
require 'fluent/test'
|
|
2
|
-
require 'fluent/test/helpers'
|
|
3
|
-
require 'fluent/test/driver/filter'
|
|
4
|
-
require "helper"
|
|
5
|
-
require "fluent/plugin/filter_jsonsizelimit.rb"
|
|
6
|
-
require 'test/unit'
|
|
7
|
-
|
|
8
|
-
class JsonSizeLimitFilterTest < Test::Unit::TestCase
|
|
9
|
-
include Fluent::Test::Helpers
|
|
10
|
-
|
|
11
|
-
def setup
|
|
12
|
-
Fluent::Test.setup
|
|
13
|
-
@tag = 'test'
|
|
14
|
-
@driver = Fluent::Test::Driver::Filter.new(Fluent::Plugin::JsonSizeLimitFilter).configure(config)
|
|
15
|
-
end
|
|
16
|
-
|
|
17
|
-
def config
|
|
18
|
-
%(
|
|
19
|
-
max_size 262000
|
|
20
|
-
max_attempts 10
|
|
21
|
-
)
|
|
22
|
-
end
|
|
23
|
-
|
|
24
|
-
test 'reduce large json size to within limits' do
|
|
25
|
-
record = { 'large_field' => 'a' * 300000 }
|
|
26
|
-
@driver.run do
|
|
27
|
-
@driver.feed(@tag, Fluent::EventTime.now, record)
|
|
28
|
-
end
|
|
29
|
-
filtered_records = @driver.filtered_records
|
|
30
|
-
|
|
31
|
-
assert_true filtered_records[0].to_json.bytesize <= 262000
|
|
32
|
-
end
|
|
33
|
-
|
|
34
|
-
test 'retain smaller json size' do
|
|
35
|
-
record = { 'small_field' => 'Hello, world!' }
|
|
36
|
-
@driver.run do
|
|
37
|
-
@driver.feed(@tag, Fluent::EventTime.now, record)
|
|
38
|
-
end
|
|
39
|
-
filtered_records = @driver.filtered_records
|
|
40
|
-
|
|
41
|
-
assert_equal record.to_json.bytesize, filtered_records[0].to_json.bytesize
|
|
42
|
-
end
|
|
43
|
-
end
|