fluent-plugin-logdna 0.1.6 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/.circleci/config.yml +126 -0
- data/.gitignore +5 -4
- data/.rubocop.yml +136 -0
- data/CONTRIBUTING.md +15 -0
- data/Gemfile +8 -0
- data/LICENSE +2 -2
- data/README.md +66 -36
- data/Rakefile +54 -0
- data/fluent-plugin-logdna.gemspec +17 -15
- data/lib/fluent/plugin/out_logdna.rb +48 -17
- data/test/helper.rb +10 -0
- data/test/plugin/test_out_logdna.rb +109 -0
- metadata +43 -10
- data/.ruby-version +0 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
|
-
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
2
|
+
SHA256:
|
|
3
|
+
metadata.gz: f3c61cb20aee4655e2214d113b957d810e9347f48faed5c0e25d11b17756f619
|
|
4
|
+
data.tar.gz: 0ee054acb8ecffe663aee5e30ad56602d79b2020d9e7fd6762b084bb05cad303
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 8b78ca28b43d342694fa585c7fe3dbafdb993bacc8db24701c5ae955adb3291c0d12a8e83a84e082a74d2d2c2e2cf77df951975a80c912ce5f429cec7126a26a
|
|
7
|
+
data.tar.gz: 75898267356efe81538acfb82abeacefc566e7bd667c1bb263b84b113d4838e0d9f828a8105d183a1c655171638d74cc2ecebd913f2b2a59a3060ff5403f8441
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
version: 2.1
|
|
2
|
+
tagged_build_filters: &tagged_build_filters
|
|
3
|
+
branches:
|
|
4
|
+
ignore: /.*/
|
|
5
|
+
tags:
|
|
6
|
+
only: /v[0-9]+\.[0-9]+\.[0-9]+/
|
|
7
|
+
test_build_filters: &test_build_filters
|
|
8
|
+
branches:
|
|
9
|
+
only: /.*/
|
|
10
|
+
tags:
|
|
11
|
+
ignore: /v[0-9]+\.[0-9]+\.[0-9]+/
|
|
12
|
+
executors:
|
|
13
|
+
releaser:
|
|
14
|
+
docker:
|
|
15
|
+
- image: circleci/golang:1.12
|
|
16
|
+
ruby265:
|
|
17
|
+
docker:
|
|
18
|
+
- image: circleci/ruby:2.6.5
|
|
19
|
+
ruby23:
|
|
20
|
+
docker:
|
|
21
|
+
- image: circleci/ruby:2.3
|
|
22
|
+
jobs:
|
|
23
|
+
test:
|
|
24
|
+
parameters:
|
|
25
|
+
ruby:
|
|
26
|
+
type: executor
|
|
27
|
+
executor: << parameters.ruby >>
|
|
28
|
+
steps:
|
|
29
|
+
- checkout
|
|
30
|
+
- run: bundle install
|
|
31
|
+
- run: rubocop -c .rubocop.yml
|
|
32
|
+
- run: rake test
|
|
33
|
+
build:
|
|
34
|
+
executor: ruby265
|
|
35
|
+
steps:
|
|
36
|
+
- checkout
|
|
37
|
+
- run:
|
|
38
|
+
name: Check Tagged Push
|
|
39
|
+
command: |
|
|
40
|
+
PKG_VERSION=$(cat fluent-plugin-logdna.gemspec | grep "\.version" | cut -d'"' -f2)
|
|
41
|
+
if [[ "${CIRCLE_TAG}" != "v${PKG_VERSION}" ]]; then
|
|
42
|
+
echo "There is mismatch:"
|
|
43
|
+
echo " TAG_VERSION: ${CIRCLE_TAG}"
|
|
44
|
+
echo " PKG_VERSION: v${PKG_VERSION}"
|
|
45
|
+
exit 1
|
|
46
|
+
fi
|
|
47
|
+
- run: gem build fluent-plugin-logdna.gemspec
|
|
48
|
+
- persist_to_workspace:
|
|
49
|
+
root: .
|
|
50
|
+
paths:
|
|
51
|
+
- ./fluent-plugin-logdna-*.gem
|
|
52
|
+
release:
|
|
53
|
+
executor: releaser
|
|
54
|
+
steps:
|
|
55
|
+
- attach_workspace:
|
|
56
|
+
at: .
|
|
57
|
+
- run: go get -u github.com/tcnksm/ghr
|
|
58
|
+
- run:
|
|
59
|
+
name: Create a Release
|
|
60
|
+
command: |
|
|
61
|
+
ghr \
|
|
62
|
+
-n "LogDNA Ruby Logger ${CIRCLE_TAG}" \
|
|
63
|
+
-t ${GITHUB_TOKEN} \
|
|
64
|
+
-u ${CIRCLE_PROJECT_USERNAME} \
|
|
65
|
+
-r ${CIRCLE_PROJECT_REPONAME} \
|
|
66
|
+
-draft ${CIRCLE_TAG} ./
|
|
67
|
+
- persist_to_workspace:
|
|
68
|
+
root: .
|
|
69
|
+
paths:
|
|
70
|
+
- ./fluent-plugin-logdna-*.gem
|
|
71
|
+
approve:
|
|
72
|
+
machine: true
|
|
73
|
+
steps:
|
|
74
|
+
- attach_workspace:
|
|
75
|
+
at: .
|
|
76
|
+
- persist_to_workspace:
|
|
77
|
+
root: .
|
|
78
|
+
paths:
|
|
79
|
+
- ./fluent-plugin-logdna-*.gem
|
|
80
|
+
publish:
|
|
81
|
+
executor: ruby265
|
|
82
|
+
steps:
|
|
83
|
+
- attach_workspace:
|
|
84
|
+
at: .
|
|
85
|
+
- run:
|
|
86
|
+
name: Setup gem credentials
|
|
87
|
+
command: |
|
|
88
|
+
mkdir -p ~/.gem
|
|
89
|
+
cat \<<EOF > ~/.gem/credentials
|
|
90
|
+
---
|
|
91
|
+
:rubygems_api_key: ${RUBYGEMS_API_KEY}
|
|
92
|
+
EOF
|
|
93
|
+
chmod 0600 ~/.gem/credentials
|
|
94
|
+
- run: gem push *.gem
|
|
95
|
+
workflows:
|
|
96
|
+
update:
|
|
97
|
+
jobs:
|
|
98
|
+
- test:
|
|
99
|
+
matrix:
|
|
100
|
+
parameters:
|
|
101
|
+
ruby: [ruby23, ruby265]
|
|
102
|
+
filters: *tagged_build_filters
|
|
103
|
+
- build:
|
|
104
|
+
requires:
|
|
105
|
+
- test
|
|
106
|
+
filters: *tagged_build_filters
|
|
107
|
+
- release:
|
|
108
|
+
requires:
|
|
109
|
+
- build
|
|
110
|
+
filters: *tagged_build_filters
|
|
111
|
+
- approve:
|
|
112
|
+
type: approval
|
|
113
|
+
requires:
|
|
114
|
+
- release
|
|
115
|
+
filters: *tagged_build_filters
|
|
116
|
+
- publish:
|
|
117
|
+
requires:
|
|
118
|
+
- approve
|
|
119
|
+
filters: *tagged_build_filters
|
|
120
|
+
test:
|
|
121
|
+
jobs:
|
|
122
|
+
- test:
|
|
123
|
+
matrix:
|
|
124
|
+
parameters:
|
|
125
|
+
ruby: [ruby23, ruby265]
|
|
126
|
+
filters: *test_build_filters
|
data/.gitignore
CHANGED
|
@@ -42,11 +42,12 @@ build-iPhoneSimulator/
|
|
|
42
42
|
|
|
43
43
|
# for a library or gem, you might want to ignore these files since the code is
|
|
44
44
|
# intended to run in multiple environments; otherwise, check them in:
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
45
|
+
Gemfile.lock
|
|
46
|
+
.ruby-version
|
|
47
|
+
.ruby-gemset
|
|
48
48
|
|
|
49
49
|
# unless supporting rvm < 1.11.0 or doing something fancy, ignore this:
|
|
50
50
|
.rvmrc
|
|
51
51
|
|
|
52
|
-
.DS_Store
|
|
52
|
+
.DS_Store
|
|
53
|
+
Makefile
|
data/.rubocop.yml
ADDED
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
# Too short methods lead to extraction of single-use methods, which can make
|
|
2
|
+
# the code easier to read (by naming things), but can also clutter the class
|
|
3
|
+
Metrics/MethodLength:
|
|
4
|
+
Max: 100
|
|
5
|
+
|
|
6
|
+
# The guiding principle of classes is SRP, SRP can't be accurately measured by LoC
|
|
7
|
+
Metrics/ClassLength:
|
|
8
|
+
Max: 1500
|
|
9
|
+
|
|
10
|
+
Metrics/AbcSize:
|
|
11
|
+
Max: 40
|
|
12
|
+
|
|
13
|
+
Metrics/CyclomaticComplexity:
|
|
14
|
+
Max: 40
|
|
15
|
+
|
|
16
|
+
Metrics/PerceivedComplexity:
|
|
17
|
+
Max: 40
|
|
18
|
+
|
|
19
|
+
# Commonly used screens these days easily fit more than 80 characters.
|
|
20
|
+
Layout/LineLength:
|
|
21
|
+
Max: 120
|
|
22
|
+
|
|
23
|
+
# Single quotes being faster is hardly measurable and only affects parse time.
|
|
24
|
+
# Enforcing double quotes reduces the times where you need to change them
|
|
25
|
+
# when introducing an interpolation. Use single quotes only if their semantics
|
|
26
|
+
# are needed.
|
|
27
|
+
Style/StringLiterals:
|
|
28
|
+
EnforcedStyle: double_quotes
|
|
29
|
+
|
|
30
|
+
# We do not need to support Ruby 1.9, so this is good to use.
|
|
31
|
+
Style/SymbolArray:
|
|
32
|
+
Enabled: true
|
|
33
|
+
|
|
34
|
+
# Mixing the styles looks just silly.
|
|
35
|
+
Style/HashSyntax:
|
|
36
|
+
EnforcedStyle: ruby19_no_mixed_keys
|
|
37
|
+
|
|
38
|
+
# has_key? and has_value? are far more readable than key? and value?
|
|
39
|
+
Style/PreferredHashMethods:
|
|
40
|
+
Enabled: false
|
|
41
|
+
|
|
42
|
+
# String#% is by far the least verbose and only object oriented variant.
|
|
43
|
+
Style/FormatString:
|
|
44
|
+
EnforcedStyle: percent
|
|
45
|
+
|
|
46
|
+
Style/CollectionMethods:
|
|
47
|
+
Enabled: true
|
|
48
|
+
PreferredMethods:
|
|
49
|
+
# inject seems more common in the community.
|
|
50
|
+
reduce: "inject"
|
|
51
|
+
|
|
52
|
+
Style/RescueStandardError:
|
|
53
|
+
Enabled: false
|
|
54
|
+
|
|
55
|
+
# Either allow this style or don't. Marking it as safe with parenthesis
|
|
56
|
+
# is silly. Let's try to live without them for now.
|
|
57
|
+
Style/ParenthesesAroundCondition:
|
|
58
|
+
AllowSafeAssignment: false
|
|
59
|
+
|
|
60
|
+
# A specialized exception class will take one or more arguments and construct the message from it.
|
|
61
|
+
# So both variants make sense.
|
|
62
|
+
Style/RaiseArgs:
|
|
63
|
+
Enabled: false
|
|
64
|
+
|
|
65
|
+
# Fail is an alias of raise. Avoid aliases, it's more cognitive load for no gain.
|
|
66
|
+
# The argument that fail should be used to abort the program is wrong too,
|
|
67
|
+
# there's Kernel#abort for that.
|
|
68
|
+
Style/SignalException:
|
|
69
|
+
EnforcedStyle: only_raise
|
|
70
|
+
|
|
71
|
+
Style/GuardClause:
|
|
72
|
+
Enabled: false
|
|
73
|
+
|
|
74
|
+
# { ... } for multi-line blocks is okay, follow Weirichs rule instead:
|
|
75
|
+
# https://web.archive.org/web/20140221124509/http://onestepback.org/index.cgi/Tech/Ruby/BraceVsDoEnd.rdoc
|
|
76
|
+
Style/BlockDelimiters:
|
|
77
|
+
Enabled: false
|
|
78
|
+
|
|
79
|
+
# do / end blocks should be used for side effects,
|
|
80
|
+
# methods that run a block for side effects and have
|
|
81
|
+
# a useful return value are rare, assign the return
|
|
82
|
+
# value to a local variable for those cases.
|
|
83
|
+
Style/MethodCalledOnDoEndBlock:
|
|
84
|
+
Enabled: true
|
|
85
|
+
|
|
86
|
+
# Enforcing the names of variables? To single letter ones? Just no.
|
|
87
|
+
Style/SingleLineBlockParams:
|
|
88
|
+
Enabled: false
|
|
89
|
+
|
|
90
|
+
# Shadowing outer local variables with block parameters is often useful
|
|
91
|
+
# to not reinvent a new name for the same thing, it highlights the relation
|
|
92
|
+
# between the outer variable and the parameter. The cases where it's actually
|
|
93
|
+
# confusing are rare, and usually bad for other reasons already, for example
|
|
94
|
+
# because the method is too long.
|
|
95
|
+
Lint/ShadowingOuterLocalVariable:
|
|
96
|
+
Enabled: false
|
|
97
|
+
|
|
98
|
+
Lint/AssignmentInCondition:
|
|
99
|
+
Enabled: false
|
|
100
|
+
|
|
101
|
+
# Check with yard instead.
|
|
102
|
+
Style/Documentation:
|
|
103
|
+
Enabled: false
|
|
104
|
+
|
|
105
|
+
# This is just silly. Calling the argument `other` in all cases makes no sense.
|
|
106
|
+
Naming/BinaryOperatorParameterName:
|
|
107
|
+
Enabled: false
|
|
108
|
+
|
|
109
|
+
# There are valid cases, for example debugging Cucumber steps,
|
|
110
|
+
# also they'll fail CI anyway
|
|
111
|
+
Lint/Debugger:
|
|
112
|
+
Enabled: false
|
|
113
|
+
|
|
114
|
+
Security/Eval:
|
|
115
|
+
Enabled: false
|
|
116
|
+
# Style preference
|
|
117
|
+
Style/MethodDefParentheses:
|
|
118
|
+
Enabled: false
|
|
119
|
+
|
|
120
|
+
Style/TrailingCommaInHashLiteral:
|
|
121
|
+
Enabled: false
|
|
122
|
+
|
|
123
|
+
Style/IfUnlessModifier:
|
|
124
|
+
Enabled: false
|
|
125
|
+
|
|
126
|
+
Lint/DuplicateMethods:
|
|
127
|
+
Enabled: true
|
|
128
|
+
|
|
129
|
+
Style/RedundantSelf:
|
|
130
|
+
Enabled: false
|
|
131
|
+
|
|
132
|
+
Style/NegatedIf:
|
|
133
|
+
Enabled: false
|
|
134
|
+
|
|
135
|
+
Style/SafeNavigation:
|
|
136
|
+
Enabled: false
|
data/CONTRIBUTING.md
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
# Contributing
|
|
2
|
+
|
|
3
|
+
## Github Workflow
|
|
4
|
+
|
|
5
|
+
Contributions are always welcome! Be sure to follow the [github workflow](https://guides.github.com/introduction/flow/) when contributing to this project:
|
|
6
|
+
|
|
7
|
+
* Create an issue, or comment on an issue to indicate what you are working on. This avoids work duplication.
|
|
8
|
+
* Fork the repository and clone to your local machine
|
|
9
|
+
* You should already be on the default branch `master` - if not, check it out (`git checkout master`)
|
|
10
|
+
* Create a new branch for your feature/fix `git checkout -b my-new-feature`)
|
|
11
|
+
* Write your feature/fix
|
|
12
|
+
* Stage the changed files for a commit (`git add .`)
|
|
13
|
+
* Commit your files with a *useful* commit message ([example](https://github.com/Azure/azure-quickstart-templates/commit/53699fed9983d4adead63d9182566dec4b8430d4)) (`git commit`)
|
|
14
|
+
* Push your new branch to your GitHub Fork (`git push origin my-new-feature`)
|
|
15
|
+
* Visit this repository in GitHub and create a Pull Request.
|
data/Gemfile
ADDED
data/LICENSE
CHANGED
data/README.md
CHANGED
|
@@ -6,54 +6,84 @@ Using fluent-plugin-logdna, you can send the logs you collect with Fluentd to Lo
|
|
|
6
6
|
|
|
7
7
|
## Instructions
|
|
8
8
|
|
|
9
|
-
*
|
|
10
|
-
* `
|
|
11
|
-
*
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
9
|
+
* Requirements:
|
|
10
|
+
* `ruby >= 2.3`
|
|
11
|
+
* `fluentd < 2.0`
|
|
12
|
+
* Install Fluentd
|
|
13
|
+
* [Download here](http://www.fluentd.org/download)
|
|
14
|
+
* If using fluentd package manager (td-agent): `td-agent-gem install fluent-plugin-logdna`
|
|
15
|
+
* To install without td-agent: `gem install fluent-plugin-logdna`
|
|
16
|
+
* Add the config below to `/etc/fluent/fluent.conf`. For `td-agent`, `/etc/td-agent/td-agent.conf`:
|
|
17
|
+
|
|
18
|
+
## Configuration
|
|
19
|
+
|
|
20
|
+
### Configuration Parameters
|
|
21
|
+
- `api_key`: [Ingestion Key](https://docs.logdna.com/docs/ingestion-key), *Required*
|
|
22
|
+
- `hostname`: Hostname, *Required*
|
|
23
|
+
- `app`: App Name, *Optional*
|
|
24
|
+
- `mac`: MAC Address, *Optional*
|
|
25
|
+
- `ip`: IP Address, *Optional*
|
|
26
|
+
- `tags`: Comma-Separated List of Tags, *Optional*
|
|
27
|
+
- `request_timeout`: HTTPS POST Request Timeout, *Optional*
|
|
28
|
+
- **Note**: Supports `s` and `ms` Suffices
|
|
29
|
+
- **Default**: `30 s`
|
|
30
|
+
- `ingester_domain`: Custom Ingester URL, *Optional*
|
|
31
|
+
- **Default**: `htttps://logs.logdna.com`
|
|
32
|
+
- `ingester_endpoint`: Custom Ingester Endpoint, *Optional*
|
|
33
|
+
- **Default**: `/logs/ingest`
|
|
34
|
+
|
|
35
|
+
### Sample Configuration
|
|
36
|
+
|
|
37
|
+
~~~~~configuration
|
|
38
|
+
<match **>
|
|
15
39
|
@type logdna
|
|
16
|
-
api_key xxxxxxxxxxxxxxxxxxxxxxxxxxx
|
|
17
|
-
hostname "#{Socket.gethostname}"
|
|
18
|
-
app my_app
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
40
|
+
api_key xxxxxxxxxxxxxxxxxxxxxxxxxxx
|
|
41
|
+
hostname "#{Socket.gethostname}"
|
|
42
|
+
app my_app
|
|
43
|
+
mac C0:FF:EE:C0:FF:EE
|
|
44
|
+
ip 127.0.0.1
|
|
45
|
+
tags web,dev
|
|
46
|
+
request_timeout 30000 ms
|
|
47
|
+
ingester_domain https://logs.logdna.com
|
|
23
48
|
</match>
|
|
24
49
|
~~~~~
|
|
25
|
-
* Restart fluentd to pick up the configuration changes.
|
|
26
50
|
|
|
27
|
-
|
|
51
|
+
## Line Parameters
|
|
28
52
|
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
- We recommend increasing these values if you are encountering problems.
|
|
53
|
+
The following line parameters can be set to the information coming from each `record` object:
|
|
54
|
+
- `level`: [Level](https://github.com/logdna/logger-node#supported-log-levels): `record['level']` or `record['severity']` or the last `tag` given in each `record`
|
|
55
|
+
- `file`: File Name: set to `file` given in each `record`
|
|
56
|
+
- `app`: App Name: set to either `_app` or `app` given in each `record`
|
|
57
|
+
- **Default**: `app` given in the configuration
|
|
58
|
+
- `env`: Environment Name: set to `env` given in each `record`
|
|
59
|
+
- `meta`: Meta Object: set to `meta` given in each `record`
|
|
37
60
|
|
|
38
|
-
###
|
|
61
|
+
### LogDNA Pay-per-gig Pricing
|
|
39
62
|
|
|
40
|
-
|
|
41
|
-
* `_app` and `level` will override the config
|
|
63
|
+
Our [paid plans](https://logdna.com/pricing/) start at $1.25/GB per month, and it's based only on usage. There are no fixed data buckets and all paid plans include all features.
|
|
42
64
|
|
|
43
|
-
|
|
65
|
+
## Building a debian package for td-agent
|
|
44
66
|
|
|
45
|
-
|
|
46
|
-
brew cask install logdna-cli
|
|
47
|
-
logdna register <email>
|
|
48
|
-
# now paste the api key above
|
|
49
|
-
```
|
|
67
|
+
If you use td-agent you can build a debian package instead of installing via `td-agent-gem`. This requires that td-agent is already installed and that you've installed [fpm](http://fpm.readthedocs.io/en/latest/index.html). Then run `make` in your git directory.
|
|
50
68
|
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
69
|
+
~~~~~bash
|
|
70
|
+
gem install --no-document fpm
|
|
71
|
+
git clone https://github.com/logdna/fluent-plugin-logdna
|
|
72
|
+
cd fluent-plugin-logdna
|
|
73
|
+
gem build fluent-plugin-logdna.gemspec
|
|
74
|
+
fpm --input-type gem --output-type deb \
|
|
75
|
+
--no-auto-depends \
|
|
76
|
+
--no-gem-fix-name \
|
|
77
|
+
--depends 'td-agent > 2' \
|
|
78
|
+
--deb-build-depends 'td-agent > 2' \
|
|
79
|
+
fluent-plugin-logdna-*.gem
|
|
80
|
+
sudo dpkg -i fluent-plugin-logdna*.deb
|
|
81
|
+
~~~~~
|
|
54
82
|
|
|
55
83
|
## Additional Options
|
|
56
84
|
|
|
57
|
-
For advanced configuration options, please refer to the [buffered output parameters documentation.](
|
|
85
|
+
For advanced configuration options, please refer to the [buffered output parameters documentation.](https://docs.fluentd.org/v/0.12/output#buffered-output-parameters)
|
|
58
86
|
|
|
59
87
|
Questions or concerns? Contact [support@logdna.com](mailto:support@logdna.com).
|
|
88
|
+
|
|
89
|
+
Contributions are always welcome. See the [contributing guide](/CONTRIBUTING.md) to learn how you can help.
|
data/Rakefile
ADDED
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "set"
|
|
4
|
+
require "rake/clean"
|
|
5
|
+
require "rake/testtask"
|
|
6
|
+
|
|
7
|
+
# Directory to place gems to bundle with our main gem
|
|
8
|
+
PACKAGE_DIR = "pkg"
|
|
9
|
+
CLOBBER.include PACKAGE_DIR
|
|
10
|
+
|
|
11
|
+
desc "Copies gem dependencies to the #{PACKAGE_DIR} folder"
|
|
12
|
+
task "gemdep" do
|
|
13
|
+
gem_copy gen_gem_list
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
desc "Copies gem dependencies which don't overlap with td-agent to the #{PACKAGE_DIR} folder"
|
|
17
|
+
task "gemdep:minimal" do
|
|
18
|
+
MINIMAL_GEMS = %w[
|
|
19
|
+
unf_ext
|
|
20
|
+
unf
|
|
21
|
+
domain_name
|
|
22
|
+
http-cookie
|
|
23
|
+
http-form_data
|
|
24
|
+
http
|
|
25
|
+
].freeze
|
|
26
|
+
gem_copy gen_gem_list MINIMAL_GEMS
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def gen_gem_list(subset = [])
|
|
30
|
+
myself = Bundler.load_gemspec(Dir[File.join(Bundler::SharedHelpers.pwd, "{,*}.gemspec")].first)
|
|
31
|
+
to_pull = Set[]
|
|
32
|
+
Bundler.locked_gems.specs.each do |spec|
|
|
33
|
+
next if spec.name == myself.name
|
|
34
|
+
|
|
35
|
+
if subset.empty?
|
|
36
|
+
to_pull.add(spec)
|
|
37
|
+
elsif subset.include? spec.name
|
|
38
|
+
to_pull.add(spec)
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
to_pull
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# This assumes that bundle install has already been run (which populates
|
|
45
|
+
# Bundler.app_cache with the gems of dependencies).
|
|
46
|
+
def gem_copy(specs)
|
|
47
|
+
Bundler.mkdir_p PACKAGE_DIR
|
|
48
|
+
sources = specs.map { |s| Bundler.app_cache.join "#{s.full_name}.gem" }
|
|
49
|
+
FileUtils.cp sources, PACKAGE_DIR, verbose: true
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
Rake::TestTask.new do |task|
|
|
53
|
+
task.pattern = "test/**/*.rb"
|
|
54
|
+
end
|
|
@@ -1,22 +1,24 @@
|
|
|
1
|
-
#
|
|
2
|
-
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "date"
|
|
3
4
|
|
|
4
5
|
Gem::Specification.new do |s|
|
|
5
|
-
s.name =
|
|
6
|
-
s.version =
|
|
6
|
+
s.name = "fluent-plugin-logdna"
|
|
7
|
+
s.version = "0.4.0"
|
|
7
8
|
s.date = Date.today.to_s
|
|
8
|
-
s.summary =
|
|
9
|
-
s.description =
|
|
10
|
-
s.authors = [
|
|
11
|
-
s.email =
|
|
12
|
-
s.
|
|
13
|
-
s.
|
|
14
|
-
s.license = 'MIT'
|
|
9
|
+
s.summary = "LogDNA Plugin for Fluentd"
|
|
10
|
+
s.description = "Fluentd Plugin for Supplying Output to LogDNA."
|
|
11
|
+
s.authors = ["LogDNA, Inc."]
|
|
12
|
+
s.email = "help@logdna.com"
|
|
13
|
+
s.homepage = "https://github.com/logdna/fluent-plugin-logdna"
|
|
14
|
+
s.license = "MIT"
|
|
15
15
|
|
|
16
|
-
s.require_paths = [
|
|
16
|
+
s.require_paths = ["lib"]
|
|
17
17
|
s.files = `git ls-files -z`.split("\x0")
|
|
18
18
|
s.executables = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
|
|
19
|
-
s.required_ruby_version = Gem::Requirement.new(
|
|
20
|
-
s.add_runtime_dependency
|
|
21
|
-
s.add_runtime_dependency
|
|
19
|
+
s.required_ruby_version = Gem::Requirement.new(">= 2.3")
|
|
20
|
+
s.add_runtime_dependency "fluentd", ">= 0.12.0", "< 2"
|
|
21
|
+
s.add_runtime_dependency "http", "~> 2.0", ">= 2.0.3"
|
|
22
|
+
s.add_development_dependency "bundler", "~> 1.16"
|
|
23
|
+
s.add_development_dependency "rubocop", "~> 0.78"
|
|
22
24
|
end
|
|
@@ -1,30 +1,51 @@
|
|
|
1
|
-
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fluent/output"
|
|
2
4
|
|
|
3
5
|
module Fluent
|
|
4
6
|
class LogDNAOutput < Fluent::BufferedOutput
|
|
5
|
-
Fluent::Plugin.register_output(
|
|
7
|
+
Fluent::Plugin.register_output("logdna", self)
|
|
6
8
|
|
|
7
|
-
INGESTER_DOMAIN = 'https://logs.logdna.com'.freeze
|
|
8
9
|
MAX_RETRIES = 5
|
|
9
10
|
|
|
10
11
|
config_param :api_key, :string, secret: true
|
|
11
12
|
config_param :hostname, :string
|
|
12
13
|
config_param :mac, :string, default: nil
|
|
14
|
+
config_param :tags, :string, default: nil
|
|
13
15
|
config_param :ip, :string, default: nil
|
|
14
16
|
config_param :app, :string, default: nil
|
|
17
|
+
config_param :file, :string, default: nil
|
|
18
|
+
config_param :ingester_domain, :string, default: "https://logs.logdna.com"
|
|
19
|
+
config_param :ingester_endpoint, :string, default: "/logs/ingest"
|
|
20
|
+
config_param :request_timeout, :string, default: "30"
|
|
15
21
|
|
|
16
22
|
def configure(conf)
|
|
17
23
|
super
|
|
18
|
-
@host = conf[
|
|
24
|
+
@host = conf["hostname"]
|
|
25
|
+
|
|
26
|
+
# make these two variables globals
|
|
27
|
+
timeout_unit_map = { s: 1.0, ms: 0.001 }
|
|
28
|
+
timeout_regex = Regexp.new("^([0-9]+)\s*(#{timeout_unit_map.keys.join('|')})$")
|
|
29
|
+
|
|
30
|
+
# this section goes into this part of the code
|
|
31
|
+
num_component = 30.0
|
|
32
|
+
unit_component = "s"
|
|
33
|
+
|
|
34
|
+
timeout_regex.match(@request_timeout) do |match|
|
|
35
|
+
num_component = match[1].to_f
|
|
36
|
+
unit_component = match[2]
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
@request_timeout = num_component * timeout_unit_map[unit_component.to_sym]
|
|
19
40
|
end
|
|
20
41
|
|
|
21
42
|
def start
|
|
22
43
|
super
|
|
23
|
-
require
|
|
24
|
-
require
|
|
25
|
-
require
|
|
26
|
-
HTTP.default_options = { :
|
|
27
|
-
@ingester = HTTP.persistent
|
|
44
|
+
require "json"
|
|
45
|
+
require "base64"
|
|
46
|
+
require "http"
|
|
47
|
+
HTTP.default_options = { keep_alive_timeout: 60 }
|
|
48
|
+
@ingester = HTTP.persistent @ingester_domain
|
|
28
49
|
@requests = Queue.new
|
|
29
50
|
end
|
|
30
51
|
|
|
@@ -40,7 +61,8 @@ module Fluent
|
|
|
40
61
|
def write(chunk)
|
|
41
62
|
body = chunk_to_body(chunk)
|
|
42
63
|
response = send_request(body)
|
|
43
|
-
raise
|
|
64
|
+
raise "Encountered server error" if response.code >= 400
|
|
65
|
+
|
|
44
66
|
response.flush
|
|
45
67
|
end
|
|
46
68
|
|
|
@@ -51,7 +73,7 @@ module Fluent
|
|
|
51
73
|
|
|
52
74
|
chunk.msgpack_each do |(tag, time, record)|
|
|
53
75
|
line = gather_line_data(tag, time, record)
|
|
54
|
-
data << line
|
|
76
|
+
data << line unless line[:line].empty?
|
|
55
77
|
end
|
|
56
78
|
|
|
57
79
|
{ lines: data }
|
|
@@ -59,21 +81,30 @@ module Fluent
|
|
|
59
81
|
|
|
60
82
|
def gather_line_data(tag, time, record)
|
|
61
83
|
line = {
|
|
62
|
-
level: record[
|
|
84
|
+
level: record["level"] || record["severity"] || tag.split(".").last,
|
|
63
85
|
timestamp: time,
|
|
64
|
-
line: record
|
|
86
|
+
line: record.to_json
|
|
65
87
|
}
|
|
66
|
-
|
|
88
|
+
# At least one of "file" or "app" is required.
|
|
89
|
+
line[:file] = record["file"]
|
|
90
|
+
line[:file] ||= @file if @file
|
|
91
|
+
line.delete(:file) if line[:file].nil?
|
|
92
|
+
line[:app] = record["_app"] || record["app"]
|
|
67
93
|
line[:app] ||= @app if @app
|
|
68
94
|
line.delete(:app) if line[:app].nil?
|
|
95
|
+
line[:env] = record["env"]
|
|
96
|
+
line.delete(:env) if line[:env].nil?
|
|
97
|
+
line[:meta] = record["meta"]
|
|
98
|
+
line.delete(:meta) if line[:meta].nil?
|
|
69
99
|
line
|
|
70
100
|
end
|
|
71
101
|
|
|
72
102
|
def send_request(body)
|
|
73
103
|
now = Time.now.to_i
|
|
74
|
-
url = "
|
|
75
|
-
@ingester.headers(
|
|
76
|
-
|
|
104
|
+
url = "#{@ingester_endpoint}?hostname=#{@host}&mac=#{@mac}&ip=#{@ip}&now=#{now}&tags=#{@tags}"
|
|
105
|
+
@ingester.headers("apikey" => @api_key,
|
|
106
|
+
"content-type" => "application/json")
|
|
107
|
+
.timeout(connect: @request_timeout, write: @request_timeout, read: @request_timeout)
|
|
77
108
|
.post(url, json: body)
|
|
78
109
|
end
|
|
79
110
|
end
|
data/test/helper.rb
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
$LOAD_PATH.unshift(File.expand_path("..", __dir__))
|
|
4
|
+
require "test-unit"
|
|
5
|
+
require "fluent/test"
|
|
6
|
+
require "fluent/test/driver/output"
|
|
7
|
+
require "fluent/test/helpers"
|
|
8
|
+
|
|
9
|
+
Test::Unit::TestCase.include(Fluent::Test::Helpers)
|
|
10
|
+
Test::Unit::TestCase.extend(Fluent::Test::Helpers)
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# https://docs.fluentd.org/plugin-development/api-plugin-output#how-to-write-tests
|
|
4
|
+
|
|
5
|
+
require_relative "../helper"
|
|
6
|
+
require "fluent/test/driver/output"
|
|
7
|
+
require "fluent/plugin/out_stdout"
|
|
8
|
+
require "fluent/plugin/input"
|
|
9
|
+
require "webmock/test_unit"
|
|
10
|
+
|
|
11
|
+
require "lib/fluent/plugin/out_logdna.rb"
|
|
12
|
+
|
|
13
|
+
class LogdnaOutputTest < Test::Unit::TestCase
|
|
14
|
+
include WebMock::API
|
|
15
|
+
|
|
16
|
+
def setup
|
|
17
|
+
Fluent::Test.setup
|
|
18
|
+
|
|
19
|
+
stub_request(:post, %r{logs.logdna.com/logs/ingest})
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# default configuration for tests
|
|
23
|
+
CONFIG = %(
|
|
24
|
+
api_key this-is-my-key
|
|
25
|
+
hostname "localhost"
|
|
26
|
+
app my_app
|
|
27
|
+
mac C0:FF:EE:C0:FF:EE
|
|
28
|
+
ip 127.0.0.1
|
|
29
|
+
tags "my-tag"
|
|
30
|
+
)
|
|
31
|
+
|
|
32
|
+
def create_driver(conf = CONFIG)
|
|
33
|
+
Fluent::Test::Driver::Output.new(Fluent::LogDNAOutput).configure(conf)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
test "instantiate the plugin and check default config values" do
|
|
37
|
+
d = create_driver
|
|
38
|
+
|
|
39
|
+
# check defaults
|
|
40
|
+
assert_equal "https://logs.logdna.com", d.instance.ingester_domain
|
|
41
|
+
assert_equal 30, d.instance.request_timeout
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
test "instantiate the plugin and check setting config values" do
|
|
45
|
+
conf = %(
|
|
46
|
+
api_key this-is-my-key
|
|
47
|
+
hostname "localhost"
|
|
48
|
+
app my_app
|
|
49
|
+
mac C0:FF:EE:C0:FF:EE
|
|
50
|
+
ip 127.0.0.1
|
|
51
|
+
tags "my-tag"
|
|
52
|
+
request_timeout 17s
|
|
53
|
+
ingester_endpoint this/is/my/alternate/endpoint
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
d = create_driver(conf)
|
|
57
|
+
|
|
58
|
+
# check set config values
|
|
59
|
+
assert_equal "my-tag", d.instance.tags
|
|
60
|
+
assert_equal 17, d.instance.request_timeout
|
|
61
|
+
assert_equal "this/is/my/alternate/endpoint", d.instance.ingester_endpoint
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
test "instantiate the plugin with ms request_timeout value" do
|
|
65
|
+
conf = %(
|
|
66
|
+
api_key this-is-my-key
|
|
67
|
+
hostname "localhost"
|
|
68
|
+
app my_app
|
|
69
|
+
mac C0:FF:EE:C0:FF:EE
|
|
70
|
+
ip 127.0.0.1
|
|
71
|
+
tags "my-tag"
|
|
72
|
+
request_timeout 17000 ms
|
|
73
|
+
)
|
|
74
|
+
|
|
75
|
+
d = create_driver(conf)
|
|
76
|
+
|
|
77
|
+
# check set config values
|
|
78
|
+
assert_equal 17, d.instance.request_timeout
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
test "instantiate the plugin with nonesense request_timeout value" do
|
|
82
|
+
conf = %(
|
|
83
|
+
api_key this-is-my-key
|
|
84
|
+
hostname "localhost"
|
|
85
|
+
app my_app
|
|
86
|
+
mac C0:FF:EE:C0:FF:EE
|
|
87
|
+
ip 127.0.0.1
|
|
88
|
+
tags "my-tag"
|
|
89
|
+
request_timeout "asdf ms"
|
|
90
|
+
)
|
|
91
|
+
|
|
92
|
+
d = create_driver(conf)
|
|
93
|
+
|
|
94
|
+
# check set config values
|
|
95
|
+
assert_equal 30, d.instance.request_timeout
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
test "simple #write" do
|
|
99
|
+
d = create_driver
|
|
100
|
+
time = event_time
|
|
101
|
+
|
|
102
|
+
d.run do
|
|
103
|
+
d.feed("output.test", time, { "foo" => "bar", "message" => "myLine" })
|
|
104
|
+
d.feed("output.test", time, { "foo" => "bar", "message" => "myLine" })
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
assert_equal(2, d.formatted.size)
|
|
108
|
+
end
|
|
109
|
+
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-logdna
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.4.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
|
-
-
|
|
7
|
+
- LogDNA, Inc.
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2020-09-17 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: fluentd
|
|
@@ -50,18 +50,52 @@ dependencies:
|
|
|
50
50
|
- - ">="
|
|
51
51
|
- !ruby/object:Gem::Version
|
|
52
52
|
version: 2.0.3
|
|
53
|
-
|
|
54
|
-
|
|
53
|
+
- !ruby/object:Gem::Dependency
|
|
54
|
+
name: bundler
|
|
55
|
+
requirement: !ruby/object:Gem::Requirement
|
|
56
|
+
requirements:
|
|
57
|
+
- - "~>"
|
|
58
|
+
- !ruby/object:Gem::Version
|
|
59
|
+
version: '1.16'
|
|
60
|
+
type: :development
|
|
61
|
+
prerelease: false
|
|
62
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
63
|
+
requirements:
|
|
64
|
+
- - "~>"
|
|
65
|
+
- !ruby/object:Gem::Version
|
|
66
|
+
version: '1.16'
|
|
67
|
+
- !ruby/object:Gem::Dependency
|
|
68
|
+
name: rubocop
|
|
69
|
+
requirement: !ruby/object:Gem::Requirement
|
|
70
|
+
requirements:
|
|
71
|
+
- - "~>"
|
|
72
|
+
- !ruby/object:Gem::Version
|
|
73
|
+
version: '0.78'
|
|
74
|
+
type: :development
|
|
75
|
+
prerelease: false
|
|
76
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
77
|
+
requirements:
|
|
78
|
+
- - "~>"
|
|
79
|
+
- !ruby/object:Gem::Version
|
|
80
|
+
version: '0.78'
|
|
81
|
+
description: Fluentd Plugin for Supplying Output to LogDNA.
|
|
82
|
+
email: help@logdna.com
|
|
55
83
|
executables: []
|
|
56
84
|
extensions: []
|
|
57
85
|
extra_rdoc_files: []
|
|
58
86
|
files:
|
|
87
|
+
- ".circleci/config.yml"
|
|
59
88
|
- ".gitignore"
|
|
60
|
-
- ".
|
|
89
|
+
- ".rubocop.yml"
|
|
90
|
+
- CONTRIBUTING.md
|
|
91
|
+
- Gemfile
|
|
61
92
|
- LICENSE
|
|
62
93
|
- README.md
|
|
94
|
+
- Rakefile
|
|
63
95
|
- fluent-plugin-logdna.gemspec
|
|
64
96
|
- lib/fluent/plugin/out_logdna.rb
|
|
97
|
+
- test/helper.rb
|
|
98
|
+
- test/plugin/test_out_logdna.rb
|
|
65
99
|
homepage: https://github.com/logdna/fluent-plugin-logdna
|
|
66
100
|
licenses:
|
|
67
101
|
- MIT
|
|
@@ -74,16 +108,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
74
108
|
requirements:
|
|
75
109
|
- - ">="
|
|
76
110
|
- !ruby/object:Gem::Version
|
|
77
|
-
version: 2.
|
|
111
|
+
version: '2.3'
|
|
78
112
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
79
113
|
requirements:
|
|
80
114
|
- - ">="
|
|
81
115
|
- !ruby/object:Gem::Version
|
|
82
116
|
version: '0'
|
|
83
117
|
requirements: []
|
|
84
|
-
|
|
85
|
-
rubygems_version: 2.6.8
|
|
118
|
+
rubygems_version: 3.0.3
|
|
86
119
|
signing_key:
|
|
87
120
|
specification_version: 4
|
|
88
|
-
summary: LogDNA
|
|
121
|
+
summary: LogDNA Plugin for Fluentd
|
|
89
122
|
test_files: []
|
data/.ruby-version
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
2.3.1
|