logstash-codec-cloudwatch_logs 0.0.1 → 0.0.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +75 -1
- data/lib/logstash/codecs/cloudwatch_logs.rb +13 -5
- data/logstash-codec-cloudwatch_logs.gemspec +7 -3
- data/spec/codecs/cloudwatch_logs_spec.rb +6 -6
- metadata +52 -10
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 1fe34dd5f76336b9bcd3bfefb5d741c4a0cf60e3
|
4
|
+
data.tar.gz: c47f73f3e7332ccf6956ae23d0b4daa58a710929
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 6da618f7c472d645e72a1a3796a63fc40e334d12e41b4aefb756c27a8d3f59309255f6016a2f343c706feb355caae6899ec7c4797fc449d411dedc1f965f0cf3
|
7
|
+
data.tar.gz: 02e9f55c7eb968e1d99b6142d78509e124030481c86166b9026b945d94ede0749bcb09061655a87498e08a4b671f0f6a24abf4c3a16f2183b3b918b95ee005b8
|
data/README.md
CHANGED
@@ -2,10 +2,20 @@
|
|
2
2
|
|
3
3
|
[![Travis Build Status](https://travis-ci.org/threadwaste/logstash-codec-cloudwatch_logs.svg)](https://travis-ci.org/threadwaste/logstash-codec-cloudwatch_logs)
|
4
4
|
|
5
|
-
Parse [CloudWatch Logs subscriptions](http://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/Subscriptions.html#DestinationKinesisExample)
|
5
|
+
Parse [CloudWatch Logs subscriptions](http://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/Subscriptions.html#DestinationKinesisExample) into individual events.
|
6
|
+
|
7
|
+
## Installation
|
8
|
+
|
9
|
+
This plugin can be installed by Logstash's plugin tool.
|
10
|
+
|
11
|
+
```
|
12
|
+
bin/logstash-plugin install logstash-codec-cloudwatch_logs
|
13
|
+
```
|
6
14
|
|
7
15
|
## Usage
|
8
16
|
|
17
|
+
At its simplest:
|
18
|
+
|
9
19
|
```
|
10
20
|
input {
|
11
21
|
kinesis {
|
@@ -14,3 +24,67 @@ input {
|
|
14
24
|
}
|
15
25
|
}
|
16
26
|
```
|
27
|
+
|
28
|
+
### Event Format
|
29
|
+
|
30
|
+
The CloudWatch Logs codec breaks each multi-event subscription record into
|
31
|
+
individual events. It does this by iterating over the `logEvents` field, and
|
32
|
+
merging each event with all other top-level fields. The codec drops the
|
33
|
+
`logEvents` field from the final event.
|
34
|
+
|
35
|
+
For example, given a subscription record:
|
36
|
+
|
37
|
+
```
|
38
|
+
{
|
39
|
+
"owner": "123456789012",
|
40
|
+
"logGroup": "Example",
|
41
|
+
"logStream": "Example1",
|
42
|
+
"subscriptionFilters": [
|
43
|
+
"RootAccess"
|
44
|
+
],
|
45
|
+
"messageType": "DATA_MESSAGE",
|
46
|
+
"logEvents": [
|
47
|
+
{
|
48
|
+
"id": "1",
|
49
|
+
"timestamp": 1478014822000,
|
50
|
+
"message": "event1"
|
51
|
+
},
|
52
|
+
{
|
53
|
+
"id": "2",
|
54
|
+
"timestamp": 1478014825000,
|
55
|
+
"message": "event2"
|
56
|
+
}
|
57
|
+
]
|
58
|
+
}
|
59
|
+
```
|
60
|
+
|
61
|
+
...this codec would yield two individual events:
|
62
|
+
|
63
|
+
```
|
64
|
+
[
|
65
|
+
{
|
66
|
+
"owner": "123456789012",
|
67
|
+
"logGroup": "Example",
|
68
|
+
"logStream": "Example1",
|
69
|
+
"subscriptionFilters": [
|
70
|
+
"RootAccess"
|
71
|
+
],
|
72
|
+
"messageType": "DATA_MESSAGE",
|
73
|
+
"id": "1",
|
74
|
+
"timestamp": 1478014822000,
|
75
|
+
"message": "event1"
|
76
|
+
},
|
77
|
+
{
|
78
|
+
"owner": "123456789012",
|
79
|
+
"logGroup": "Example",
|
80
|
+
"logStream": "Example1",
|
81
|
+
"subscriptionFilters": [
|
82
|
+
"RootAccess"
|
83
|
+
],
|
84
|
+
"messageType": "DATA_MESSAGE",
|
85
|
+
"id": "2",
|
86
|
+
"timestamp": 1478014825000,
|
87
|
+
"message": "event2"
|
88
|
+
}
|
89
|
+
]
|
90
|
+
```
|
@@ -1,5 +1,7 @@
|
|
1
1
|
# encoding: utf-8
|
2
|
-
require
|
2
|
+
require 'logstash/codecs/base'
|
3
|
+
require 'logstash/timestamp'
|
4
|
+
require 'logstash/event'
|
3
5
|
require 'logstash/json'
|
4
6
|
require 'zlib'
|
5
7
|
|
@@ -8,11 +10,15 @@ require 'zlib'
|
|
8
10
|
class LogStash::Codecs::CloudWatchLogs < LogStash::Codecs::Base
|
9
11
|
config_name "cloudwatch_logs"
|
10
12
|
|
13
|
+
# Disable the gzip decompression phase if your events have already
|
14
|
+
# been decompressed by the input processor.
|
15
|
+
config :decompress, :validate => :boolean, :default => true
|
16
|
+
|
11
17
|
public
|
12
18
|
def register; end
|
13
19
|
|
14
20
|
def decode(data, &block)
|
15
|
-
data = decompress(StringIO.new(data))
|
21
|
+
data = decompress(StringIO.new(data)) if @decompress
|
16
22
|
parse(LogStash::Json.load(data), &block)
|
17
23
|
end
|
18
24
|
|
@@ -25,11 +31,13 @@ class LogStash::Codecs::CloudWatchLogs < LogStash::Codecs::Base
|
|
25
31
|
end
|
26
32
|
|
27
33
|
def parse(json, &block)
|
28
|
-
|
29
|
-
|
34
|
+
events = json.delete("logEvents")
|
35
|
+
json.freeze
|
30
36
|
|
31
37
|
events.each do |event|
|
32
|
-
|
38
|
+
epochmillis = event.delete("timestamp").to_i
|
39
|
+
event[LogStash::Event::TIMESTAMP] = LogStash::Timestamp.at(epochmillis / 1000, (epochmillis % 1000) * 1000)
|
40
|
+
yield LogStash::Event.new(json.merge(event))
|
33
41
|
end
|
34
42
|
end
|
35
43
|
end
|
@@ -1,12 +1,12 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
s.name = 'logstash-codec-cloudwatch_logs'
|
3
|
-
s.version = '0.0.
|
3
|
+
s.version = '0.0.2'
|
4
4
|
s.licenses = ['Apache License (2.0)']
|
5
5
|
s.summary = "Parse CloudWatch Logs subscription data"
|
6
6
|
s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
|
7
7
|
s.authors = ["Anthony M."]
|
8
8
|
s.email = 'tony@threadwaste.com'
|
9
|
-
s.homepage = "https://github.com/threadwaste/logstash-codec-
|
9
|
+
s.homepage = "https://github.com/threadwaste/logstash-codec-cloudwatch_logs"
|
10
10
|
s.require_paths = ["lib"]
|
11
11
|
|
12
12
|
# Files
|
@@ -19,7 +19,11 @@ Gem::Specification.new do |s|
|
|
19
19
|
s.metadata = { "logstash_plugin" => "true", "logstash_group" => "codec" }
|
20
20
|
|
21
21
|
# Gem dependencies
|
22
|
-
s.add_runtime_dependency
|
22
|
+
s.add_runtime_dependency 'logstash-core-plugin-api', '>= 1.60', '<= 2.99'
|
23
|
+
s.add_runtime_dependency 'cabin', '~> 0.6'
|
23
24
|
|
24
25
|
s.add_development_dependency 'logstash-devutils', '>= 0.0.16'
|
26
|
+
s.add_development_dependency 'logstash-codec-json'
|
27
|
+
s.add_development_dependency 'tins', '1.6'
|
28
|
+
|
25
29
|
end
|
@@ -33,14 +33,14 @@ describe LogStash::Codecs::CloudWatchLogs do
|
|
33
33
|
expect(events.size).to eq 3
|
34
34
|
|
35
35
|
events.each do |event|
|
36
|
-
expect(event
|
37
|
-
expect(event
|
38
|
-
expect(event
|
39
|
-
expect(event
|
40
|
-
expect(event
|
36
|
+
expect(event.get('owner')).to eq '123456789012'
|
37
|
+
expect(event.get('logGroup')).to eq 'CloudTrail'
|
38
|
+
expect(event.get('logStream')).to eq '123456789012_CloudTrail_us-east-1'
|
39
|
+
expect(event.get('subscriptionFilters')).to eq ['RootAccess']
|
40
|
+
expect(event.get('messageType')).to eq 'DATA_MESSAGE'
|
41
41
|
end
|
42
42
|
|
43
|
-
messages = events.map { |e| e
|
43
|
+
messages = events.map { |e| e.get("message") }
|
44
44
|
expect(messages).to eq ["first", "second", "third"]
|
45
45
|
end
|
46
46
|
end
|
metadata
CHANGED
@@ -1,35 +1,49 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-codec-cloudwatch_logs
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.2
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Anthony M.
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-11-
|
11
|
+
date: 2016-11-14 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
requirement: !ruby/object:Gem::Requirement
|
15
15
|
requirements:
|
16
16
|
- - ">="
|
17
17
|
- !ruby/object:Gem::Version
|
18
|
-
version:
|
19
|
-
- - "
|
18
|
+
version: '1.60'
|
19
|
+
- - "<="
|
20
20
|
- !ruby/object:Gem::Version
|
21
|
-
version:
|
22
|
-
name: logstash-core
|
21
|
+
version: '2.99'
|
22
|
+
name: logstash-core-plugin-api
|
23
23
|
prerelease: false
|
24
24
|
type: :runtime
|
25
25
|
version_requirements: !ruby/object:Gem::Requirement
|
26
26
|
requirements:
|
27
27
|
- - ">="
|
28
28
|
- !ruby/object:Gem::Version
|
29
|
-
version:
|
30
|
-
- - "
|
29
|
+
version: '1.60'
|
30
|
+
- - "<="
|
31
31
|
- !ruby/object:Gem::Version
|
32
|
-
version:
|
32
|
+
version: '2.99'
|
33
|
+
- !ruby/object:Gem::Dependency
|
34
|
+
requirement: !ruby/object:Gem::Requirement
|
35
|
+
requirements:
|
36
|
+
- - "~>"
|
37
|
+
- !ruby/object:Gem::Version
|
38
|
+
version: '0.6'
|
39
|
+
name: cabin
|
40
|
+
prerelease: false
|
41
|
+
type: :runtime
|
42
|
+
version_requirements: !ruby/object:Gem::Requirement
|
43
|
+
requirements:
|
44
|
+
- - "~>"
|
45
|
+
- !ruby/object:Gem::Version
|
46
|
+
version: '0.6'
|
33
47
|
- !ruby/object:Gem::Dependency
|
34
48
|
requirement: !ruby/object:Gem::Requirement
|
35
49
|
requirements:
|
@@ -44,6 +58,34 @@ dependencies:
|
|
44
58
|
- - ">="
|
45
59
|
- !ruby/object:Gem::Version
|
46
60
|
version: 0.0.16
|
61
|
+
- !ruby/object:Gem::Dependency
|
62
|
+
requirement: !ruby/object:Gem::Requirement
|
63
|
+
requirements:
|
64
|
+
- - ">="
|
65
|
+
- !ruby/object:Gem::Version
|
66
|
+
version: '0'
|
67
|
+
name: logstash-codec-json
|
68
|
+
prerelease: false
|
69
|
+
type: :development
|
70
|
+
version_requirements: !ruby/object:Gem::Requirement
|
71
|
+
requirements:
|
72
|
+
- - ">="
|
73
|
+
- !ruby/object:Gem::Version
|
74
|
+
version: '0'
|
75
|
+
- !ruby/object:Gem::Dependency
|
76
|
+
requirement: !ruby/object:Gem::Requirement
|
77
|
+
requirements:
|
78
|
+
- - '='
|
79
|
+
- !ruby/object:Gem::Version
|
80
|
+
version: '1.6'
|
81
|
+
name: tins
|
82
|
+
prerelease: false
|
83
|
+
type: :development
|
84
|
+
version_requirements: !ruby/object:Gem::Requirement
|
85
|
+
requirements:
|
86
|
+
- - '='
|
87
|
+
- !ruby/object:Gem::Version
|
88
|
+
version: '1.6'
|
47
89
|
description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
|
48
90
|
email: tony@threadwaste.com
|
49
91
|
executables: []
|
@@ -56,7 +98,7 @@ files:
|
|
56
98
|
- lib/logstash/codecs/cloudwatch_logs.rb
|
57
99
|
- logstash-codec-cloudwatch_logs.gemspec
|
58
100
|
- spec/codecs/cloudwatch_logs_spec.rb
|
59
|
-
homepage: https://github.com/threadwaste/logstash-codec-
|
101
|
+
homepage: https://github.com/threadwaste/logstash-codec-cloudwatch_logs
|
60
102
|
licenses:
|
61
103
|
- Apache License (2.0)
|
62
104
|
metadata:
|