fluent-plugin-cloudwatch-ingest 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 8f7d5923cde41024c0c47da234c11e928c4f1e20
|
|
4
|
+
data.tar.gz: 3fe64dbc099cb12f0a1419add3f5b92ff33b6bec
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: ce09d0fd283d29189fd50e503caeee7904482e205c27926993fda0649b2eb5fbae313fc49a919ce0a364c3e875068b7948879839db00864fe488b7f118a47d00
|
|
7
|
+
data.tar.gz: 4f8d9aeb782b1569ef3cccb1edffd232e02e2fef6f311ddc8224fc43d1165574e3e7cf70737d4fd681e34f5b8d67aaf437e7df3c2131f6cde31b74d6c88546a8
|
data/README.md
CHANGED
|
@@ -47,9 +47,11 @@ Or install it yourself as:
|
|
|
47
47
|
@type cloudwatch_ingest
|
|
48
48
|
expression /^(?<message>.+)$/
|
|
49
49
|
time_format %Y-%m-%d %H:%M:%S.%L
|
|
50
|
-
event_time true
|
|
51
|
-
inject_group_name true
|
|
52
|
-
inject_stream_name true
|
|
50
|
+
event_time true # take time from the Cloudwatch event, rather than parse it from the body
|
|
51
|
+
inject_group_name true # inject the group name into the record
|
|
52
|
+
inject_stream_name true # inject the stream name into the record
|
|
53
|
+
parse_json_body false # Attempt to parse the body as json and add structured fields from the result
|
|
54
|
+
fail_on_unparsable_json false # If the body cannot be parsed as json do not ingest the record
|
|
53
55
|
</parse>
|
|
54
56
|
</source>
|
|
55
57
|
```
|
|
@@ -70,6 +72,13 @@ When the state file is located on a shared filesystem an exclusive write lock wi
|
|
|
70
72
|
As such it is safe to run multiple instances of this plugin consuming from the same CloudWatch logging source without fear of duplication, as long as they share a state file.
|
|
71
73
|
In a properly configured auto-scaling group this provides for uninterrupted log ingestion in the event of a failure of any single node.
|
|
72
74
|
|
|
75
|
+
### JSON parsing
|
|
76
|
+
With the `parse_json_body` option set to `true` the plugin will attempt to parse the body of the log entry as JSON. If this is successful any field/value pairs found will be added to the emitted record as structured fields.
|
|
77
|
+
|
|
78
|
+
If `fail_on_unparsable_json` is set to `true` a record body consisting of malformed json will cause the record to be rejected. You may wish to leave this setting as false if the plugin is ingesting multiple log groups with a mixture of json/structured and unstructured content.
|
|
79
|
+
|
|
80
|
+
The `expression` is applied before JSON parsing is attempted. One may therefore extract a JSON fragment from within the event body if it is decorated with additional free-form text.
|
|
81
|
+
|
|
73
82
|
### Sub-second timestamps
|
|
74
83
|
When using `event_time true` the `@timestamp` field for the record is taken from the time recorded against the event by Cloudwatch. This is the most common mode to run in as it's an easy path to normalization: all of your Lambdas or other AWS service need not have the same, valid, `time_format` nor a regex that matches every case.
|
|
75
84
|
|
|
@@ -78,9 +87,7 @@ If your output plugin supports sub-second precision (and you're running fluentd
|
|
|
78
87
|
#### Elasticsearch
|
|
79
88
|
It is a common pattern to use fluentd alongside the [fluentd-plugin-elasticsearch](https://github.com/uken/fluent-plugin-elasticsearch) plugin, either directly or via [fluent-plugin-aws-elasticsearch-service](https://github.com/atomita/fluent-plugin-aws-elasticsearch-service), to ingest logs into Elasticsearch.
|
|
80
89
|
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
Failing that I maintain my own fork of that repository with the fix in place: https://github.com/sampointer/fluent-plugin-elasticsearch/tree/add_configurable_time_precision_when_timestamp_missing
|
|
90
|
+
Prior to version 1.9.5 there was a bug within that plugin which, via an unwise cast, caused records without a named timestamp field to be cast to `DateTime`, losing the precision. This PR: https://github.com/uken/fluent-plugin-elasticsearch/pull/249 fixed that issue.
|
|
84
91
|
|
|
85
92
|
### IAM
|
|
86
93
|
IAM is a tricky and often bespoke subject. Here's a starter that will ingest all of the logs for all of your Lambdas in the account in which the plugin is running:
|
|
@@ -116,8 +123,7 @@ IAM is a tricky and often bespoke subject. Here's a starter that will ingest all
|
|
|
116
123
|
```
|
|
117
124
|
|
|
118
125
|
### Cross-account authentication
|
|
119
|
-
|
|
120
|
-
needs to be able to `sts:AssumeRole` the `sts_arn` (and obviously needs `sts_enabled` to be true).
|
|
126
|
+
Broadly speaking the IAM instance role of the host on which the plugin is running needs to be able to `sts:AssumeRole` the `sts_arn` (and obviously needs `sts_enabled` to be true).
|
|
121
127
|
|
|
122
128
|
The assumed role should look more-or-less like that above in terms of the actions and resource combinations required.
|
|
123
129
|
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
require 'fluent/plugin/parser_regexp'
|
|
2
2
|
require 'fluent/time'
|
|
3
|
+
require 'multi_json'
|
|
3
4
|
|
|
4
5
|
module Fluent
|
|
5
6
|
module Plugin
|
|
@@ -11,6 +12,8 @@ module Fluent
|
|
|
11
12
|
config_param :event_time, :bool, default: true
|
|
12
13
|
config_param :inject_group_name, :bool, default: true
|
|
13
14
|
config_param :inject_stream_name, :bool, default: true
|
|
15
|
+
config_param :parse_json_body, :bool, default: false
|
|
16
|
+
config_param :fail_on_unparsable_json, :bool, default: false
|
|
14
17
|
|
|
15
18
|
def initialize
|
|
16
19
|
super
|
|
@@ -28,6 +31,18 @@ module Fluent
|
|
|
28
31
|
record = r
|
|
29
32
|
end
|
|
30
33
|
|
|
34
|
+
# Optionally attempt to parse the body as json
|
|
35
|
+
if @parse_json_body
|
|
36
|
+
begin
|
|
37
|
+
record.merge!(MultiJson.load(record))
|
|
38
|
+
rescue MultiJson::ParseError
|
|
39
|
+
if @fail_on_unparsable_json
|
|
40
|
+
yield nil, nil
|
|
41
|
+
return
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
|
|
31
46
|
# Inject optional fields
|
|
32
47
|
record['log_group_name'] = group if @inject_group_name
|
|
33
48
|
record['log_stream_name'] = stream if @inject_stream_name
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: fluent-plugin-cloudwatch-ingest
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.5.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Sam Pointer
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2017-
|
|
11
|
+
date: 2017-06-15 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|
|
@@ -94,6 +94,20 @@ dependencies:
|
|
|
94
94
|
- - "~>"
|
|
95
95
|
- !ruby/object:Gem::Version
|
|
96
96
|
version: 2.8.4
|
|
97
|
+
- !ruby/object:Gem::Dependency
|
|
98
|
+
name: multi_json
|
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
|
100
|
+
requirements:
|
|
101
|
+
- - "~>"
|
|
102
|
+
- !ruby/object:Gem::Version
|
|
103
|
+
version: '1.12'
|
|
104
|
+
type: :runtime
|
|
105
|
+
prerelease: false
|
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
107
|
+
requirements:
|
|
108
|
+
- - "~>"
|
|
109
|
+
- !ruby/object:Gem::Version
|
|
110
|
+
version: '1.12'
|
|
97
111
|
description: Fluentd plugin to ingest AWS Cloudwatch logs
|
|
98
112
|
email:
|
|
99
113
|
- san@outsidethe.net
|