logstash-codec-cloudwatch_logs 0.0.1 → 0.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +75 -1
  3. data/lib/logstash/codecs/cloudwatch_logs.rb +13 -5
  4. data/logstash-codec-cloudwatch_logs.gemspec +7 -3
  5. data/spec/codecs/cloudwatch_logs_spec.rb +6 -6
  6. metadata +52 -10
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: c93c61a0f6290abeaa6435db7b5abd7eaea0e431
4
- data.tar.gz: 2f24162de1251bad7eeeb8dfcaf0bc40de815dab
3
+ metadata.gz: 1fe34dd5f76336b9bcd3bfefb5d741c4a0cf60e3
4
+ data.tar.gz: c47f73f3e7332ccf6956ae23d0b4daa58a710929
5
5
  SHA512:
6
- metadata.gz: a942cf6850c35d2675bbad4304168157134a5c0883cc2b1350f4e2516ee199575a8f3be1da913f17804a45c1242f2510d27a0a7f30b6e9120d69af739e1dc8fc
7
- data.tar.gz: 75b69c66dde11669ee7b97d05e1473f3e2873a5fcd17708392b76575e18f74f636bac0741726fe310132444d9f1b632e26cd1f2626616937733c15a8dd8c43b6
6
+ metadata.gz: 6da618f7c472d645e72a1a3796a63fc40e334d12e41b4aefb756c27a8d3f59309255f6016a2f343c706feb355caae6899ec7c4797fc449d411dedc1f965f0cf3
7
+ data.tar.gz: 02e9f55c7eb968e1d99b6142d78509e124030481c86166b9026b945d94ede0749bcb09061655a87498e08a4b671f0f6a24abf4c3a16f2183b3b918b95ee005b8
data/README.md CHANGED
@@ -2,10 +2,20 @@
2
2
 
3
3
  [![Travis Build Status](https://travis-ci.org/threadwaste/logstash-codec-cloudwatch_logs.svg)](https://travis-ci.org/threadwaste/logstash-codec-cloudwatch_logs)
4
4
 
5
- Parse [CloudWatch Logs subscriptions](http://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/Subscriptions.html#DestinationKinesisExample) sent to Kinesis.
5
+ Parse [CloudWatch Logs subscriptions](http://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/Subscriptions.html#DestinationKinesisExample) into individual events.
6
+
7
+ ## Installation
8
+
9
+ This plugin can be installed by Logstash's plugin tool.
10
+
11
+ ```
12
+ bin/logstash-plugin install logstash-codec-cloudwatch_logs
13
+ ```
6
14
 
7
15
  ## Usage
8
16
 
17
+ At its simplest:
18
+
9
19
  ```
10
20
  input {
11
21
  kinesis {
@@ -14,3 +24,67 @@ input {
14
24
  }
15
25
  }
16
26
  ```
27
+
28
+ ### Event Format
29
+
30
+ The CloudWatch Logs codec breaks each multi-event subscription record into
31
+ individual events. It does this by iterating over the `logEvents` field, and
32
+ merging each event with all other top-level fields. The codec drops the
33
+ `logEvents` field from the final event.
34
+
35
+ For example, given a subscription record:
36
+
37
+ ```
38
+ {
39
+ "owner": "123456789012",
40
+ "logGroup": "Example",
41
+ "logStream": "Example1",
42
+ "subscriptionFilters": [
43
+ "RootAccess"
44
+ ],
45
+ "messageType": "DATA_MESSAGE",
46
+ "logEvents": [
47
+ {
48
+ "id": "1",
49
+ "timestamp": 1478014822000,
50
+ "message": "event1"
51
+ },
52
+ {
53
+ "id": "2",
54
+ "timestamp": 1478014825000,
55
+ "message": "event2"
56
+ }
57
+ ]
58
+ }
59
+ ```
60
+
61
+ ...this codec would yield two individual events:
62
+
63
+ ```
64
+ [
65
+ {
66
+ "owner": "123456789012",
67
+ "logGroup": "Example",
68
+ "logStream": "Example1",
69
+ "subscriptionFilters": [
70
+ "RootAccess"
71
+ ],
72
+ "messageType": "DATA_MESSAGE",
73
+ "id": "1",
74
+ "timestamp": 1478014822000,
75
+ "message": "event1"
76
+ },
77
+ {
78
+ "owner": "123456789012",
79
+ "logGroup": "Example",
80
+ "logStream": "Example1",
81
+ "subscriptionFilters": [
82
+ "RootAccess"
83
+ ],
84
+ "messageType": "DATA_MESSAGE",
85
+ "id": "2",
86
+ "timestamp": 1478014825000,
87
+ "message": "event2"
88
+ }
89
+ ]
90
+ ```
data/lib/logstash/codecs/cloudwatch_logs.rb CHANGED
@@ -1,5 +1,7 @@
1
1
  # encoding: utf-8
2
- require "logstash/codecs/base"
2
+ require 'logstash/codecs/base'
3
+ require 'logstash/timestamp'
4
+ require 'logstash/event'
3
5
  require 'logstash/json'
4
6
  require 'zlib'
5
7
 
@@ -8,11 +10,15 @@ require 'zlib'
8
10
  class LogStash::Codecs::CloudWatchLogs < LogStash::Codecs::Base
9
11
  config_name "cloudwatch_logs"
10
12
 
13
+ # Disable the gzip decompression phase if your events have already
14
+ # been decompressed by the input processor.
15
+ config :decompress, :validate => :boolean, :default => true
16
+
11
17
  public
12
18
  def register; end
13
19
 
14
20
  def decode(data, &block)
15
- data = decompress(StringIO.new(data))
21
+ data = decompress(StringIO.new(data)) if @decompress
16
22
  parse(LogStash::Json.load(data), &block)
17
23
  end
18
24
 
@@ -25,11 +31,13 @@ class LogStash::Codecs::CloudWatchLogs < LogStash::Codecs::Base
25
31
  end
26
32
 
27
33
  def parse(json, &block)
28
- base = json.reject { |k,_| k == "logEvents" }.freeze
29
- events = json["logEvents"]
34
+ events = json.delete("logEvents")
35
+ json.freeze
30
36
 
31
37
  events.each do |event|
32
- yield LogStash::Event.new(base.merge(event))
38
+ epochmillis = event.delete("timestamp").to_i
39
+ event[LogStash::Event::TIMESTAMP] = LogStash::Timestamp.at(epochmillis / 1000, (epochmillis % 1000) * 1000)
40
+ yield LogStash::Event.new(json.merge(event))
33
41
  end
34
42
  end
35
43
  end
data/logstash-codec-cloudwatch_logs.gemspec CHANGED
@@ -1,12 +1,12 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-codec-cloudwatch_logs'
3
- s.version = '0.0.1'
3
+ s.version = '0.0.2'
4
4
  s.licenses = ['Apache License (2.0)']
5
5
  s.summary = "Parse CloudWatch Logs subscription data"
6
6
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
7
7
  s.authors = ["Anthony M."]
8
8
  s.email = 'tony@threadwaste.com'
9
- s.homepage = "https://github.com/threadwaste/logstash-codec-cloudwatchlogs"
9
+ s.homepage = "https://github.com/threadwaste/logstash-codec-cloudwatch_logs"
10
10
  s.require_paths = ["lib"]
11
11
 
12
12
  # Files
@@ -19,7 +19,11 @@ Gem::Specification.new do |s|
19
19
  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "codec" }
20
20
 
21
21
  # Gem dependencies
22
- s.add_runtime_dependency "logstash-core", ">= 2.0.0", "< 3.0.0"
22
+ s.add_runtime_dependency 'logstash-core-plugin-api', '>= 1.60', '<= 2.99'
23
+ s.add_runtime_dependency 'cabin', '~> 0.6'
23
24
 
24
25
  s.add_development_dependency 'logstash-devutils', '>= 0.0.16'
26
+ s.add_development_dependency 'logstash-codec-json'
27
+ s.add_development_dependency 'tins', '1.6'
28
+
25
29
  end
data/spec/codecs/cloudwatch_logs_spec.rb CHANGED
@@ -33,14 +33,14 @@ describe LogStash::Codecs::CloudWatchLogs do
33
33
  expect(events.size).to eq 3
34
34
 
35
35
  events.each do |event|
36
- expect(event['owner']).to eq '123456789012'
37
- expect(event['logGroup']).to eq 'CloudTrail'
38
- expect(event['logStream']).to eq '123456789012_CloudTrail_us-east-1'
39
- expect(event['subscriptionFilters']).to eq ['RootAccess']
40
- expect(event['messageType']).to eq 'DATA_MESSAGE'
36
+ expect(event.get('owner')).to eq '123456789012'
37
+ expect(event.get('logGroup')).to eq 'CloudTrail'
38
+ expect(event.get('logStream')).to eq '123456789012_CloudTrail_us-east-1'
39
+ expect(event.get('subscriptionFilters')).to eq ['RootAccess']
40
+ expect(event.get('messageType')).to eq 'DATA_MESSAGE'
41
41
  end
42
42
 
43
- messages = events.map { |e| e["message"] }
43
+ messages = events.map { |e| e.get("message") }
44
44
  expect(messages).to eq ["first", "second", "third"]
45
45
  end
46
46
  end
metadata CHANGED
@@ -1,35 +1,49 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-codec-cloudwatch_logs
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.1
4
+ version: 0.0.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Anthony M.
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-11-01 00:00:00.000000000 Z
11
+ date: 2016-11-14 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
15
15
  requirements:
16
16
  - - ">="
17
17
  - !ruby/object:Gem::Version
18
- version: 2.0.0
19
- - - "<"
18
+ version: '1.60'
19
+ - - "<="
20
20
  - !ruby/object:Gem::Version
21
- version: 3.0.0
22
- name: logstash-core
21
+ version: '2.99'
22
+ name: logstash-core-plugin-api
23
23
  prerelease: false
24
24
  type: :runtime
25
25
  version_requirements: !ruby/object:Gem::Requirement
26
26
  requirements:
27
27
  - - ">="
28
28
  - !ruby/object:Gem::Version
29
- version: 2.0.0
30
- - - "<"
29
+ version: '1.60'
30
+ - - "<="
31
31
  - !ruby/object:Gem::Version
32
- version: 3.0.0
32
+ version: '2.99'
33
+ - !ruby/object:Gem::Dependency
34
+ requirement: !ruby/object:Gem::Requirement
35
+ requirements:
36
+ - - "~>"
37
+ - !ruby/object:Gem::Version
38
+ version: '0.6'
39
+ name: cabin
40
+ prerelease: false
41
+ type: :runtime
42
+ version_requirements: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - "~>"
45
+ - !ruby/object:Gem::Version
46
+ version: '0.6'
33
47
  - !ruby/object:Gem::Dependency
34
48
  requirement: !ruby/object:Gem::Requirement
35
49
  requirements:
@@ -44,6 +58,34 @@ dependencies:
44
58
  - - ">="
45
59
  - !ruby/object:Gem::Version
46
60
  version: 0.0.16
61
+ - !ruby/object:Gem::Dependency
62
+ requirement: !ruby/object:Gem::Requirement
63
+ requirements:
64
+ - - ">="
65
+ - !ruby/object:Gem::Version
66
+ version: '0'
67
+ name: logstash-codec-json
68
+ prerelease: false
69
+ type: :development
70
+ version_requirements: !ruby/object:Gem::Requirement
71
+ requirements:
72
+ - - ">="
73
+ - !ruby/object:Gem::Version
74
+ version: '0'
75
+ - !ruby/object:Gem::Dependency
76
+ requirement: !ruby/object:Gem::Requirement
77
+ requirements:
78
+ - - '='
79
+ - !ruby/object:Gem::Version
80
+ version: '1.6'
81
+ name: tins
82
+ prerelease: false
83
+ type: :development
84
+ version_requirements: !ruby/object:Gem::Requirement
85
+ requirements:
86
+ - - '='
87
+ - !ruby/object:Gem::Version
88
+ version: '1.6'
47
89
  description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
48
90
  email: tony@threadwaste.com
49
91
  executables: []
@@ -56,7 +98,7 @@ files:
56
98
  - lib/logstash/codecs/cloudwatch_logs.rb
57
99
  - logstash-codec-cloudwatch_logs.gemspec
58
100
  - spec/codecs/cloudwatch_logs_spec.rb
59
- homepage: https://github.com/threadwaste/logstash-codec-cloudwatchlogs
101
+ homepage: https://github.com/threadwaste/logstash-codec-cloudwatch_logs
60
102
  licenses:
61
103
  - Apache License (2.0)
62
104
  metadata: