fluent-plugin-moogaiops 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: d3ae0568e4bf9b9678a1daca39e7dea95730df9b
4
+ data.tar.gz: b09c5048b93059361c78fba4acb86e0413ff522c
5
+ SHA512:
6
+ metadata.gz: c43e78cf7ad52ab60c3dc56ccfcaaae40961912cce98890775f81b3c350ef1b0dd38627ac68b19021b41d9b69307941cd9e467bcae906a5ff4d2c52cfc65ffc3
7
+ data.tar.gz: b3d9a4727c1a11b142ab35c53e153c06299ca17f8817d64315a0b173aafa27524a2d6026186c65b51bcae20ba2876b754865e48f397809e1a9be3e8c293dc7ec
@@ -0,0 +1,11 @@
1
+ /.bundle/
2
+ /.yardoc
3
+ /Gemfile.lock
4
+ /_yardoc/
5
+ /coverage/
6
+ /doc/
7
+ /pkg/
8
+ /spec/reports/
9
+ /tmp/
10
+ /vendor/
11
+ /*.gem
data/Gemfile ADDED
@@ -0,0 +1,5 @@
1
+ source 'https://rubygems.org'
2
+
3
+ # Specify your gem's dependencies in fluent-plugin-moogaiops.gemspec
4
+ gem 'fluentd', '~> 0.12.0'
5
+ gemspec
@@ -0,0 +1,23 @@
1
+ The MIT License (MIT)
2
+
3
+ Copyright (c) 2017 Moogsoft
4
+
5
+ Moog AIOps is a trademark of Moogsoft Inc. All rights reserved.
6
+
7
+ Permission is hereby granted, free of charge, to any person obtaining a copy
8
+ of this software and associated documentation files (the "Software"), to deal
9
+ in the Software without restriction, including without limitation the rights
10
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11
+ copies of the Software, and to permit persons to whom the Software is
12
+ furnished to do so, subject to the following conditions:
13
+
14
+ The above copyright notice and this permission notice shall be included in
15
+ all copies or substantial portions of the Software.
16
+
17
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
23
+ THE SOFTWARE.
@@ -0,0 +1,59 @@
1
+ # fluent-plugin-moogaiops
2
+
3
+ This GEM will add an output plugin for Moog AIOps.
4
+
5
+ ## Installation
6
+
7
+ Add this line to your application's Gemfile:
8
+
9
+ ```ruby
10
+ gem 'fluent-plugin-moogaiops'
11
+ ```
12
+
13
+ And then execute:
14
+
15
+ $ bundle
16
+
17
+ Or install it yourself as:
18
+
19
+ $ gem install fluent-plugin-moogaiops
20
+
21
+ ## Usage
22
+
23
+ Example matcher to add to your fulend.conf
24
+
25
+ ```
26
+ <match system.** *.access.* error.**>
27
+ @type moogaiops
28
+ uri https://<YOUR MOOGAIOPS>.moogsoft.com/events/generic_generic1
29
+ auth <YOUR USER>:<YOUR PASSWORD>
30
+ sourcetype fluentd
31
+ location london
32
+ severity 3
33
+ </match>
34
+ ```
35
+ *uri*
36
+ Is the published endpoint accepting fluent events see your integrations and select the fluent tile to install.
37
+
38
+ *auth*
39
+ Is the username and password provided by Moogaiops when you install the REST connector for fluent
40
+
41
+ *sourcetype*
42
+ Some text to define the manager attribute in the event
43
+
44
+ *location*
45
+ Some text to identify the agent_location in the events
46
+
47
+ *severity*
48
+ A default severity (0-5) for the events produced by this matcher
49
+
50
+ The Gem will also add the **hostname** where the matcher is running and some other default information. The *tag* will be used to populate the **class** and **type**, all three are used in the signature.
51
+
52
+ ## Contributing
53
+
54
+ Bug reports and pull requests are welcome on GitHub at https://github.com/moog-stephen/fluent-plugin-moogaiops.
55
+
56
+
57
+ ## License
58
+
59
+ The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
@@ -0,0 +1,2 @@
1
+ require "bundler/gem_tasks"
2
+ task :default => :spec
@@ -0,0 +1,31 @@
1
+ # coding: utf-8
2
+ lib = File.expand_path('../lib', __FILE__)
3
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
4
+
5
+ Gem::Specification.new do |spec|
6
+ spec.name = "fluent-plugin-moogaiops"
7
+ spec.authors = ["stephen"]
8
+ spec.email = ["stephen@moogsoftcom"]
9
+
10
+ spec.summary = %q{Send Fluentd output to the Moog AIOps REST LAM}
11
+ spec.description = %q{Matcher (Output plugin) to send Fluentd events to the Moog AIOps REST LAM}
12
+ spec.homepage = "https://github.com/moog-stephen/fluent-plugin-moogaiops"
13
+ spec.license = "MIT"
14
+
15
+ spec.files = `git ls-files`.split($\)
16
+ spec.name = "fluent-plugin-moogaiops"
17
+ spec.version = '0.1.0'
18
+ spec.required_ruby_version = ">= 2.1.0"
19
+
20
+ spec.add_runtime_dependency "fluentd", [">= 0.12", "< 0.14"]
21
+ spec.add_runtime_dependency 'rest-client', "~> 2.0"
22
+ spec.add_runtime_dependency 'json', "~> 2.0"
23
+
24
+ spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
25
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
26
+ spec.bindir = "exe"
27
+ spec.require_paths = ["lib"]
28
+
29
+ spec.add_development_dependency "bundler", "~> 1.14"
30
+ spec.add_development_dependency "rake", "~> 10.0"
31
+ end
@@ -0,0 +1,153 @@
1
+ require 'fluent/output'
2
+
3
+ module Fluent
4
+ require 'rest-client'
5
+ require 'json'
6
+
7
+ class MoogAIOpsOutput < BufferedOutput
8
+ # First, register the plugin. NAME is the name of this plugin
9
+ # and identifies the plugin in the configuration file.
10
+ Fluent::Plugin.register_output('moogaiops', self)
11
+
12
+ config_param :uri, :string, :default => 'http://localhost:8888'
13
+ config_param :auth, :string, :default => 'graze:graze', secret: true
14
+ config_param :source, :string, :default => '{TAG}'
15
+ config_param :sourcetype, :string, :default => 'fluentd'
16
+ config_param :location, :string, :default => 'local'
17
+ config_param :severity, :integer, :default => 1
18
+
19
+ # This method is called before starting.
20
+ # 'conf' is a Hash that includes configuration parameters.
21
+ # If the configuration is invalid, raise Fluent::ConfigError.
22
+ def configure(conf)
23
+ super
24
+
25
+ require 'socket'
26
+ @hostname = Socket.gethostname
27
+
28
+ # You can also refer raw parameter via conf[name].
29
+ @uri = conf['uri']
30
+ $log.debug "Config URI #{@uri}"
31
+ @manager = conf['sourcetype']
32
+
33
+ case @source
34
+ when '{TAG}'
35
+ @source_formatter = lambda { |tag| tag }
36
+ else
37
+ @source_formatter = lambda { |tag| @source.sub('{TAG}', tag) }
38
+ end
39
+
40
+ @formatter = lambda { |record|
41
+ record.to_json
42
+ }
43
+
44
+ end
45
+
46
+ # This method is called when starting.
47
+ # Open sockets or files here.
48
+ def start
49
+ super
50
+ $log.debug "initialized for moogaiops"
51
+ end
52
+
53
+ # This method is called when shutting down.
54
+ # Shutdown the thread and close sockets or files here.
55
+ def shutdown
56
+ super
57
+ $log.debug "shutdown from moogaiops"
58
+ end
59
+
60
+ # This method is called when an event reaches to Fluentd.
61
+ # Convert the event to a raw string.
62
+ def format(tag, time, record)
63
+ newrecord = {}
64
+
65
+ begin
66
+ if record['severity']
67
+ newrecord['severity'] = Integer(record['severity'])
68
+ else
69
+ newrecord['severity'] = @severity
70
+ end
71
+ rescue
72
+ newrecord['severity'] = @severity
73
+ end
74
+
75
+ newrecord['type'] = tag.to_s
76
+ newrecord['agent_time'] = time.to_s
77
+ newrecord['manager'] = @manager
78
+ newrecord['class'] = @source
79
+ newrecord['source'] = @hostname
80
+ newrecord['description'] = record['message']
81
+ newrecord['custom_info'] = record
82
+
83
+ newrecord.to_msgpack
84
+ end
85
+
86
+ def chunk_to_buffers(chunk)
87
+ buffers = []
88
+ buffer = {}
89
+ events = []
90
+ chunk.msgpack_each do |event|
91
+ $log.debug "Buffering event = #{event}"
92
+ events << event
93
+ end
94
+ buffer['events'] = events
95
+ $log.debug "Return buffer #{buffer}"
96
+ buffers << buffer
97
+ return buffers
98
+ end
99
+
100
+ # This method is called every flush interval. Write the buffer chunk
101
+ # to files or databases here.
102
+ # 'chunk' is a buffer chunk that includes multiple formatted
103
+ # events. You can use 'data = chunk.read' to get all events and
104
+ # 'chunk.open {|io| ... }' to get IO objects.
105
+ #
106
+ # NOTE! This method is called by internal thread, not Fluentd's main thread. So IO wait doesn't affect other plugins.
107
+ def write(chunk)
108
+ #data = chunk.read
109
+ #print data
110
+ #$log.debug "Data #{data}"
111
+
112
+ username, password = @auth.split(':')
113
+ $log.debug "#{username} : #{password}"
114
+
115
+ chunk_to_buffers(chunk).each do |buffer|
116
+
117
+ bufj = buffer.to_json
118
+ $log.debug "Buffer #{bufj}"
119
+
120
+ re = RestClient::Resource.new(@uri, {:user => username, :password => password,:verify_ssl => 0})
121
+
122
+ response = re.post bufj, :content_type => 'application/json'
123
+ $log.debug "POST #{@uri}"
124
+ jr = JSON.parse(response.body)
125
+ $log.debug "=> #{response.code} (#{response.body})"
126
+ if response.code == 200
127
+ if jr['success']
128
+ # success
129
+ log.info "200 Ok!"
130
+ break
131
+ else
132
+ $log.error "Error 200 returned with message #{response.body}"
133
+ end
134
+ elsif response.code.match(/^40/)
135
+ # user error
136
+ $log.error "#{uri}: #{response.code} \n#{response.body}"
137
+ break
138
+ else
139
+ # other errors. fluentd will retry processing on exception
140
+ # FIXME: this may duplicate logs when using multiple buffers
141
+ raise "#{uri}: #{response.body}"
142
+ end
143
+ end
144
+ end
145
+
146
+ ## Optionally, you can use chunk.msgpack_each to deserialize objects.
147
+ #def write(chunk)
148
+ # chunk.msgpack_each {|(tag,time,record)|
149
+ # }
150
+ #end
151
+
152
+ end
153
+ end
metadata ADDED
@@ -0,0 +1,128 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: fluent-plugin-moogaiops
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - stephen
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2017-03-17 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: fluentd
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ">="
18
+ - !ruby/object:Gem::Version
19
+ version: '0.12'
20
+ - - "<"
21
+ - !ruby/object:Gem::Version
22
+ version: '0.14'
23
+ type: :runtime
24
+ prerelease: false
25
+ version_requirements: !ruby/object:Gem::Requirement
26
+ requirements:
27
+ - - ">="
28
+ - !ruby/object:Gem::Version
29
+ version: '0.12'
30
+ - - "<"
31
+ - !ruby/object:Gem::Version
32
+ version: '0.14'
33
+ - !ruby/object:Gem::Dependency
34
+ name: rest-client
35
+ requirement: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - "~>"
38
+ - !ruby/object:Gem::Version
39
+ version: '2.0'
40
+ type: :runtime
41
+ prerelease: false
42
+ version_requirements: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - "~>"
45
+ - !ruby/object:Gem::Version
46
+ version: '2.0'
47
+ - !ruby/object:Gem::Dependency
48
+ name: json
49
+ requirement: !ruby/object:Gem::Requirement
50
+ requirements:
51
+ - - "~>"
52
+ - !ruby/object:Gem::Version
53
+ version: '2.0'
54
+ type: :runtime
55
+ prerelease: false
56
+ version_requirements: !ruby/object:Gem::Requirement
57
+ requirements:
58
+ - - "~>"
59
+ - !ruby/object:Gem::Version
60
+ version: '2.0'
61
+ - !ruby/object:Gem::Dependency
62
+ name: bundler
63
+ requirement: !ruby/object:Gem::Requirement
64
+ requirements:
65
+ - - "~>"
66
+ - !ruby/object:Gem::Version
67
+ version: '1.14'
68
+ type: :development
69
+ prerelease: false
70
+ version_requirements: !ruby/object:Gem::Requirement
71
+ requirements:
72
+ - - "~>"
73
+ - !ruby/object:Gem::Version
74
+ version: '1.14'
75
+ - !ruby/object:Gem::Dependency
76
+ name: rake
77
+ requirement: !ruby/object:Gem::Requirement
78
+ requirements:
79
+ - - "~>"
80
+ - !ruby/object:Gem::Version
81
+ version: '10.0'
82
+ type: :development
83
+ prerelease: false
84
+ version_requirements: !ruby/object:Gem::Requirement
85
+ requirements:
86
+ - - "~>"
87
+ - !ruby/object:Gem::Version
88
+ version: '10.0'
89
+ description: Matcher (Output plugin) to send Fluentd events to the Moog AIOps REST
90
+ LAM
91
+ email:
92
+ - stephen@moogsoftcom
93
+ executables: []
94
+ extensions: []
95
+ extra_rdoc_files: []
96
+ files:
97
+ - ".gitignore"
98
+ - Gemfile
99
+ - LICENSE.txt
100
+ - README.md
101
+ - Rakefile
102
+ - fluent-plugin-moogaiops.gemspec
103
+ - lib/fulent/plugin/out_moogaiops.rb
104
+ homepage: https://github.com/moog-stephen/fluent-plugin-moogaiops
105
+ licenses:
106
+ - MIT
107
+ metadata: {}
108
+ post_install_message:
109
+ rdoc_options: []
110
+ require_paths:
111
+ - lib
112
+ required_ruby_version: !ruby/object:Gem::Requirement
113
+ requirements:
114
+ - - ">="
115
+ - !ruby/object:Gem::Version
116
+ version: 2.1.0
117
+ required_rubygems_version: !ruby/object:Gem::Requirement
118
+ requirements:
119
+ - - ">="
120
+ - !ruby/object:Gem::Version
121
+ version: '0'
122
+ requirements: []
123
+ rubyforge_project:
124
+ rubygems_version: 2.4.5
125
+ signing_key:
126
+ specification_version: 4
127
+ summary: Send Fluentd output to the Moog AIOps REST LAM
128
+ test_files: []