logstash-codec-cloudwatch_logs 0.0.1 → 0.0.2

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: c93c61a0f6290abeaa6435db7b5abd7eaea0e431
4
- data.tar.gz: 2f24162de1251bad7eeeb8dfcaf0bc40de815dab
3
+ metadata.gz: 1fe34dd5f76336b9bcd3bfefb5d741c4a0cf60e3
4
+ data.tar.gz: c47f73f3e7332ccf6956ae23d0b4daa58a710929
5
5
  SHA512:
6
- metadata.gz: a942cf6850c35d2675bbad4304168157134a5c0883cc2b1350f4e2516ee199575a8f3be1da913f17804a45c1242f2510d27a0a7f30b6e9120d69af739e1dc8fc
7
- data.tar.gz: 75b69c66dde11669ee7b97d05e1473f3e2873a5fcd17708392b76575e18f74f636bac0741726fe310132444d9f1b632e26cd1f2626616937733c15a8dd8c43b6
6
+ metadata.gz: 6da618f7c472d645e72a1a3796a63fc40e334d12e41b4aefb756c27a8d3f59309255f6016a2f343c706feb355caae6899ec7c4797fc449d411dedc1f965f0cf3
7
+ data.tar.gz: 02e9f55c7eb968e1d99b6142d78509e124030481c86166b9026b945d94ede0749bcb09061655a87498e08a4b671f0f6a24abf4c3a16f2183b3b918b95ee005b8
data/README.md CHANGED
@@ -2,10 +2,20 @@
2
2
 
3
3
  [![Travis Build Status](https://travis-ci.org/threadwaste/logstash-codec-cloudwatch_logs.svg)](https://travis-ci.org/threadwaste/logstash-codec-cloudwatch_logs)
4
4
 
5
- Parse [CloudWatch Logs subscriptions](http://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/Subscriptions.html#DestinationKinesisExample) sent to Kinesis.
5
+ Parse [CloudWatch Logs subscriptions](http://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/Subscriptions.html#DestinationKinesisExample) into individual events.
6
+
7
+ ## Installation
8
+
9
+ This plugin can be installed by Logstash's plugin tool.
10
+
11
+ ```
12
+ bin/logstash-plugin install logstash-codec-cloudwatch_logs
13
+ ```
6
14
 
7
15
  ## Usage
8
16
 
17
+ At its simplest:
18
+
9
19
  ```
10
20
  input {
11
21
  kinesis {
@@ -14,3 +24,67 @@ input {
14
24
  }
15
25
  }
16
26
  ```
27
+
28
+ ### Event Format
29
+
30
+ The CloudWatch Logs codec breaks each multi-event subscription record into
31
+ individual events. It does this by iterating over the `logEvents` field, and
32
+ merging each event with all other top-level fields. The codec drops the
33
+ `logEvents` field from the final event.
34
+
35
+ For example, given a subscription record:
36
+
37
+ ```
38
+ {
39
+ "owner": "123456789012",
40
+ "logGroup": "Example",
41
+ "logStream": "Example1",
42
+ "subscriptionFilters": [
43
+ "RootAccess"
44
+ ],
45
+ "messageType": "DATA_MESSAGE",
46
+ "logEvents": [
47
+ {
48
+ "id": "1",
49
+ "timestamp": 1478014822000,
50
+ "message": "event1"
51
+ },
52
+ {
53
+ "id": "2",
54
+ "timestamp": 1478014825000,
55
+ "message": "event2"
56
+ }
57
+ ]
58
+ }
59
+ ```
60
+
61
+ ...this codec would yield two individual events:
62
+
63
+ ```
64
+ [
65
+ {
66
+ "owner": "123456789012",
67
+ "logGroup": "Example",
68
+ "logStream": "Example1",
69
+ "subscriptionFilters": [
70
+ "RootAccess"
71
+ ],
72
+ "messageType": "DATA_MESSAGE",
73
+ "id": "1",
74
+ "timestamp": 1478014822000,
75
+ "message": "event1"
76
+ },
77
+ {
78
+ "owner": "123456789012",
79
+ "logGroup": "Example",
80
+ "logStream": "Example1",
81
+ "subscriptionFilters": [
82
+ "RootAccess"
83
+ ],
84
+ "messageType": "DATA_MESSAGE",
85
+ "id": "2",
86
+ "timestamp": 1478014825000,
87
+ "message": "event2"
88
+ }
89
+ ]
90
+ ```
@@ -1,5 +1,7 @@
1
1
  # encoding: utf-8
2
- require "logstash/codecs/base"
2
+ require 'logstash/codecs/base'
3
+ require 'logstash/timestamp'
4
+ require 'logstash/event'
3
5
  require 'logstash/json'
4
6
  require 'zlib'
5
7
 
@@ -8,11 +10,15 @@ require 'zlib'
8
10
  class LogStash::Codecs::CloudWatchLogs < LogStash::Codecs::Base
9
11
  config_name "cloudwatch_logs"
10
12
 
13
+ # Disable the gzip decompression phase if your events have already
14
+ # been decompressed by the input processor.
15
+ config :decompress, :validate => :boolean, :default => true
16
+
11
17
  public
12
18
  def register; end
13
19
 
14
20
  def decode(data, &block)
15
- data = decompress(StringIO.new(data))
21
+ data = decompress(StringIO.new(data)) if @decompress
16
22
  parse(LogStash::Json.load(data), &block)
17
23
  end
18
24
 
@@ -25,11 +31,13 @@ class LogStash::Codecs::CloudWatchLogs < LogStash::Codecs::Base
25
31
  end
26
32
 
27
33
  def parse(json, &block)
28
- base = json.reject { |k,_| k == "logEvents" }.freeze
29
- events = json["logEvents"]
34
+ events = json.delete("logEvents")
35
+ json.freeze
30
36
 
31
37
  events.each do |event|
32
- yield LogStash::Event.new(base.merge(event))
38
+ epochmillis = event.delete("timestamp").to_i
39
+ event[LogStash::Event::TIMESTAMP] = LogStash::Timestamp.at(epochmillis / 1000, (epochmillis % 1000) * 1000)
40
+ yield LogStash::Event.new(json.merge(event))
33
41
  end
34
42
  end
35
43
  end
@@ -1,12 +1,12 @@
1
1
  Gem::Specification.new do |s|
2
2
  s.name = 'logstash-codec-cloudwatch_logs'
3
- s.version = '0.0.1'
3
+ s.version = '0.0.2'
4
4
  s.licenses = ['Apache License (2.0)']
5
5
  s.summary = "Parse CloudWatch Logs subscription data"
6
6
  s.description = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
7
7
  s.authors = ["Anthony M."]
8
8
  s.email = 'tony@threadwaste.com'
9
- s.homepage = "https://github.com/threadwaste/logstash-codec-cloudwatchlogs"
9
+ s.homepage = "https://github.com/threadwaste/logstash-codec-cloudwatch_logs"
10
10
  s.require_paths = ["lib"]
11
11
 
12
12
  # Files
@@ -19,7 +19,11 @@ Gem::Specification.new do |s|
19
19
  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "codec" }
20
20
 
21
21
  # Gem dependencies
22
- s.add_runtime_dependency "logstash-core", ">= 2.0.0", "< 3.0.0"
22
+ s.add_runtime_dependency 'logstash-core-plugin-api', '>= 1.60', '<= 2.99'
23
+ s.add_runtime_dependency 'cabin', '~> 0.6'
23
24
 
24
25
  s.add_development_dependency 'logstash-devutils', '>= 0.0.16'
26
+ s.add_development_dependency 'logstash-codec-json'
27
+ s.add_development_dependency 'tins', '1.6'
28
+
25
29
  end
@@ -33,14 +33,14 @@ describe LogStash::Codecs::CloudWatchLogs do
33
33
  expect(events.size).to eq 3
34
34
 
35
35
  events.each do |event|
36
- expect(event['owner']).to eq '123456789012'
37
- expect(event['logGroup']).to eq 'CloudTrail'
38
- expect(event['logStream']).to eq '123456789012_CloudTrail_us-east-1'
39
- expect(event['subscriptionFilters']).to eq ['RootAccess']
40
- expect(event['messageType']).to eq 'DATA_MESSAGE'
36
+ expect(event.get('owner')).to eq '123456789012'
37
+ expect(event.get('logGroup')).to eq 'CloudTrail'
38
+ expect(event.get('logStream')).to eq '123456789012_CloudTrail_us-east-1'
39
+ expect(event.get('subscriptionFilters')).to eq ['RootAccess']
40
+ expect(event.get('messageType')).to eq 'DATA_MESSAGE'
41
41
  end
42
42
 
43
- messages = events.map { |e| e["message"] }
43
+ messages = events.map { |e| e.get("message") }
44
44
  expect(messages).to eq ["first", "second", "third"]
45
45
  end
46
46
  end
metadata CHANGED
@@ -1,35 +1,49 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-codec-cloudwatch_logs
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.1
4
+ version: 0.0.2
5
5
  platform: ruby
6
6
  authors:
7
7
  - Anthony M.
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-11-01 00:00:00.000000000 Z
11
+ date: 2016-11-14 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  requirement: !ruby/object:Gem::Requirement
15
15
  requirements:
16
16
  - - ">="
17
17
  - !ruby/object:Gem::Version
18
- version: 2.0.0
19
- - - "<"
18
+ version: '1.60'
19
+ - - "<="
20
20
  - !ruby/object:Gem::Version
21
- version: 3.0.0
22
- name: logstash-core
21
+ version: '2.99'
22
+ name: logstash-core-plugin-api
23
23
  prerelease: false
24
24
  type: :runtime
25
25
  version_requirements: !ruby/object:Gem::Requirement
26
26
  requirements:
27
27
  - - ">="
28
28
  - !ruby/object:Gem::Version
29
- version: 2.0.0
30
- - - "<"
29
+ version: '1.60'
30
+ - - "<="
31
31
  - !ruby/object:Gem::Version
32
- version: 3.0.0
32
+ version: '2.99'
33
+ - !ruby/object:Gem::Dependency
34
+ requirement: !ruby/object:Gem::Requirement
35
+ requirements:
36
+ - - "~>"
37
+ - !ruby/object:Gem::Version
38
+ version: '0.6'
39
+ name: cabin
40
+ prerelease: false
41
+ type: :runtime
42
+ version_requirements: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - "~>"
45
+ - !ruby/object:Gem::Version
46
+ version: '0.6'
33
47
  - !ruby/object:Gem::Dependency
34
48
  requirement: !ruby/object:Gem::Requirement
35
49
  requirements:
@@ -44,6 +58,34 @@ dependencies:
44
58
  - - ">="
45
59
  - !ruby/object:Gem::Version
46
60
  version: 0.0.16
61
+ - !ruby/object:Gem::Dependency
62
+ requirement: !ruby/object:Gem::Requirement
63
+ requirements:
64
+ - - ">="
65
+ - !ruby/object:Gem::Version
66
+ version: '0'
67
+ name: logstash-codec-json
68
+ prerelease: false
69
+ type: :development
70
+ version_requirements: !ruby/object:Gem::Requirement
71
+ requirements:
72
+ - - ">="
73
+ - !ruby/object:Gem::Version
74
+ version: '0'
75
+ - !ruby/object:Gem::Dependency
76
+ requirement: !ruby/object:Gem::Requirement
77
+ requirements:
78
+ - - '='
79
+ - !ruby/object:Gem::Version
80
+ version: '1.6'
81
+ name: tins
82
+ prerelease: false
83
+ type: :development
84
+ version_requirements: !ruby/object:Gem::Requirement
85
+ requirements:
86
+ - - '='
87
+ - !ruby/object:Gem::Version
88
+ version: '1.6'
47
89
  description: This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program
48
90
  email: tony@threadwaste.com
49
91
  executables: []
@@ -56,7 +98,7 @@ files:
56
98
  - lib/logstash/codecs/cloudwatch_logs.rb
57
99
  - logstash-codec-cloudwatch_logs.gemspec
58
100
  - spec/codecs/cloudwatch_logs_spec.rb
59
- homepage: https://github.com/threadwaste/logstash-codec-cloudwatchlogs
101
+ homepage: https://github.com/threadwaste/logstash-codec-cloudwatch_logs
60
102
  licenses:
61
103
  - Apache License (2.0)
62
104
  metadata: