logstash-codec-multiline 0.1.0 → 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,15 +1,7 @@
1
1
  ---
2
- !binary "U0hBMQ==":
3
- metadata.gz: !binary |-
4
- ZmMxYzRlNDc3MDQzYTVkYmE4MjY5ZTgxMDAzZWE5ZjdkNWY0NGM0OQ==
5
- data.tar.gz: !binary |-
6
- ODZiZjhhMTIxNGYyMTkzNmRlZjM4MzM5NTJjM2I4YzBlMDE5NzczOQ==
2
+ SHA1:
3
+ metadata.gz: fbc8079a193b1eaf268980c3b94c19718e95f639
4
+ data.tar.gz: 5e24a021a4aca8b06bca3eddb13a3c5c15e8f517
7
5
  SHA512:
8
- metadata.gz: !binary |-
9
- NTZiZjFkMDBjOGZjNWJjNDlkYjhmNzU3NGRlMmY3MDlhNWMwOWY2OGU0YjE3
10
- ZWU2MTBhY2IzM2ZkYzBjNmI0NDA4ZDU2ZGJiOTBmMGRiMzVlOGI4ZjQxMTkz
11
- ZjkwZDQ5NmQ1Mjg5NmIxOGRkNWI2NTA1Y2FiYTdmZWMzZjhlZGE=
12
- data.tar.gz: !binary |-
13
- NGE2YzM4ZmUwOTAxZjRkOTA5Y2IyYjk1MTcyMzE5YTFiY2FhNjU4Zjk5ZWFl
14
- ZWUxZDEyNjU1YTczOTY2ODM4MWRhNDMxM2RkZjBhNTA5OGYxYjQwNWYwNmEw
15
- NDdmZTQ4OWIyNjZlYTRjMzUwZWJkZGI5Yjg4YWY2MTUxYjVlZTc=
6
+ metadata.gz: aae90760736384020c916dbbd3f029952469a9b5910bd9e287a4c56eae94fb4b3e47d94a9264d152e4d307aeb4e62cc961e5e5e095e1489ab604cbac164be0ae
7
+ data.tar.gz: 61c3cdbc2fa91c8a861c1527fdf451ee0f19ea478e26e0df26aacdf5ddd6afa6f7f558ce0171ae2ab8b8f3e5b2df86ed2b2428a1e896b653d2d3ae6804526a97
data/Gemfile CHANGED
@@ -1,4 +1,3 @@
1
- source 'http://rubygems.org'
2
- gem 'rake'
3
- gem 'gem_publisher'
4
- gem 'archive-tar-minitar'
1
+ source 'https://rubygems.org'
2
+ gemspec
3
+ gem "logstash", :github => "elasticsearch/logstash", :branch => "1.5"
data/Rakefile CHANGED
@@ -4,3 +4,4 @@ task :default do
4
4
  system("rake -T")
5
5
  end
6
6
 
7
+ require "logstash/devutils/rake"
@@ -11,7 +11,7 @@ require "logstash/timestamp"
11
11
  # stacktrace messages into a single event.
12
12
  #
13
13
  # The config looks like this:
14
- #
14
+ # [source,ruby]
15
15
  # input {
16
16
  # stdin {
17
17
  # codec => multiline {
@@ -25,16 +25,16 @@ require "logstash/timestamp"
25
25
  # The `pattern` should match what you believe to be an indicator that the field
26
26
  # is part of a multi-line event.
27
27
  #
28
- # The `what` must be "previous" or "next" and indicates the relation
28
+ # The `what` must be `previous` or `next` and indicates the relation
29
29
  # to the multi-line event.
30
30
  #
31
- # The `negate` can be "true" or "false" (defaults to "false"). If "true", a
31
+ # The `negate` can be `true` or `false` (defaults to `false`). If `true`, a
32
32
  # message not matching the pattern will constitute a match of the multiline
33
33
  # filter and the `what` will be applied. (vice-versa is also true)
34
34
  #
35
35
  # For example, Java stack traces are multiline and usually have the message
36
36
  # starting at the far-left, with each subsequent line indented. Do this:
37
- #
37
+ # [source,ruby]
38
38
  # input {
39
39
  # stdin {
40
40
  # codec => multiline {
@@ -48,7 +48,7 @@ require "logstash/timestamp"
48
48
  #
49
49
  # Another example is to merge lines not starting with a date up to the previous
50
50
  # line..
51
- #
51
+ # [source,ruby]
52
52
  # input {
53
53
  # file {
54
54
  # path => "/var/log/someapp.log"
@@ -64,7 +64,7 @@ require "logstash/timestamp"
64
64
  # This says that any line not starting with a timestamp should be merged with the previous line.
65
65
  #
66
66
  # One more common example is C line continuations (backslash). Here's how to do that:
67
- #
67
+ # [source,ruby]
68
68
  # filter {
69
69
  # multiline {
70
70
  # type => "somefiletype"
@@ -94,21 +94,21 @@ class LogStash::Codecs::Multiline < LogStash::Codecs::Base
94
94
  # patterns.
95
95
  #
96
96
  # Pattern files are plain text with format:
97
- #
97
+ # [source,ruby]
98
98
  # NAME PATTERN
99
99
  #
100
100
  # For example:
101
- #
101
+ # [source,ruby]
102
102
  # NUMBER \d+
103
103
  config :patterns_dir, :validate => :array, :default => []
104
104
 
105
- # The character encoding used in this input. Examples include "UTF-8"
106
- # and "cp1252"
105
+ # The character encoding used in this input. Examples include `UTF-8`
106
+ # and `cp1252`
107
107
  #
108
- # This setting is useful if your log files are in Latin-1 (aka cp1252)
109
- # or in another character set other than UTF-8.
108
+ # This setting is useful if your log files are in `Latin-1` (aka `cp1252`)
109
+ # or in another character set other than `UTF-8`.
110
110
  #
111
- # This only affects "plain" format logs since JSON is UTF-8 already.
111
+ # This only affects "plain" format logs since JSON is `UTF-8` already.
112
112
  config :charset, :validate => ::Encoding.name_list, :default => "UTF-8"
113
113
 
114
114
  # Tag multiline events with a given tag. This tag will only be added
@@ -1,13 +1,13 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-codec-multiline'
4
- s.version = '0.1.0'
4
+ s.version = '0.1.1'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = "The multiline codec will collapse multiline messages and merge them into a single event."
7
- s.description = "The multiline codec will collapse multiline messages and merge them into a single event."
7
+ s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
8
8
  s.authors = ["Elasticsearch"]
9
9
  s.email = 'richard.pijnenburg@elasticsearch.com'
10
- s.homepage = "http://logstash.net/"
10
+ s.homepage = "http://www.elasticsearch.org/guide/en/logstash/current/index.html"
11
11
  s.require_paths = ["lib"]
12
12
 
13
13
  # Files
@@ -17,7 +17,7 @@ Gem::Specification.new do |s|
17
17
  s.test_files = s.files.grep(%r{^(test|spec|features)/})
18
18
 
19
19
  # Special flag to let us know this is actually a logstash plugin
20
- s.metadata = { "logstash_plugin" => "true", "group" => "codec" }
20
+ s.metadata = { "logstash_plugin" => "true", "logstash_group" => "codec" }
21
21
 
22
22
  # Gem dependencies
23
23
  s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
@@ -25,5 +25,6 @@ Gem::Specification.new do |s|
25
25
  s.add_runtime_dependency 'logstash-patterns-core'
26
26
  s.add_runtime_dependency 'jls-grok', [ '0.11.0' ]
27
27
 
28
+ s.add_development_dependency 'logstash-devutils'
28
29
  end
29
30
 
metadata CHANGED
@@ -1,65 +1,78 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-codec-multiline
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.0
4
+ version: 0.1.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Elasticsearch
8
- autorequire:
8
+ autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2014-11-05 00:00:00.000000000 Z
11
+ date: 2014-11-19 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: logstash
15
- requirement: !ruby/object:Gem::Requirement
15
+ version_requirements: !ruby/object:Gem::Requirement
16
16
  requirements:
17
- - - ! '>='
17
+ - - '>='
18
18
  - !ruby/object:Gem::Version
19
19
  version: 1.4.0
20
20
  - - <
21
21
  - !ruby/object:Gem::Version
22
22
  version: 2.0.0
23
- type: :runtime
24
- prerelease: false
25
- version_requirements: !ruby/object:Gem::Requirement
23
+ requirement: !ruby/object:Gem::Requirement
26
24
  requirements:
27
- - - ! '>='
25
+ - - '>='
28
26
  - !ruby/object:Gem::Version
29
27
  version: 1.4.0
30
28
  - - <
31
29
  - !ruby/object:Gem::Version
32
30
  version: 2.0.0
31
+ prerelease: false
32
+ type: :runtime
33
33
  - !ruby/object:Gem::Dependency
34
34
  name: logstash-patterns-core
35
- requirement: !ruby/object:Gem::Requirement
35
+ version_requirements: !ruby/object:Gem::Requirement
36
36
  requirements:
37
- - - ! '>='
37
+ - - '>='
38
38
  - !ruby/object:Gem::Version
39
39
  version: '0'
40
- type: :runtime
41
- prerelease: false
42
- version_requirements: !ruby/object:Gem::Requirement
40
+ requirement: !ruby/object:Gem::Requirement
43
41
  requirements:
44
- - - ! '>='
42
+ - - '>='
45
43
  - !ruby/object:Gem::Version
46
44
  version: '0'
45
+ prerelease: false
46
+ type: :runtime
47
47
  - !ruby/object:Gem::Dependency
48
48
  name: jls-grok
49
+ version_requirements: !ruby/object:Gem::Requirement
50
+ requirements:
51
+ - - '='
52
+ - !ruby/object:Gem::Version
53
+ version: 0.11.0
49
54
  requirement: !ruby/object:Gem::Requirement
50
55
  requirements:
51
56
  - - '='
52
57
  - !ruby/object:Gem::Version
53
58
  version: 0.11.0
54
- type: :runtime
55
59
  prerelease: false
60
+ type: :runtime
61
+ - !ruby/object:Gem::Dependency
62
+ name: logstash-devutils
56
63
  version_requirements: !ruby/object:Gem::Requirement
57
64
  requirements:
58
- - - '='
65
+ - - '>='
59
66
  - !ruby/object:Gem::Version
60
- version: 0.11.0
61
- description: The multiline codec will collapse multiline messages and merge them into
62
- a single event.
67
+ version: '0'
68
+ requirement: !ruby/object:Gem::Requirement
69
+ requirements:
70
+ - - '>='
71
+ - !ruby/object:Gem::Version
72
+ version: '0'
73
+ prerelease: false
74
+ type: :development
75
+ description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
63
76
  email: richard.pijnenburg@elasticsearch.com
64
77
  executables: []
65
78
  extensions: []
@@ -71,35 +84,32 @@ files:
71
84
  - Rakefile
72
85
  - lib/logstash/codecs/multiline.rb
73
86
  - logstash-codec-multiline.gemspec
74
- - rakelib/publish.rake
75
- - rakelib/vendor.rake
76
87
  - spec/codecs/multiline_spec.rb
77
- homepage: http://logstash.net/
88
+ homepage: http://www.elasticsearch.org/guide/en/logstash/current/index.html
78
89
  licenses:
79
90
  - Apache License (2.0)
80
91
  metadata:
81
92
  logstash_plugin: 'true'
82
- group: codec
83
- post_install_message:
93
+ logstash_group: codec
94
+ post_install_message:
84
95
  rdoc_options: []
85
96
  require_paths:
86
97
  - lib
87
98
  required_ruby_version: !ruby/object:Gem::Requirement
88
99
  requirements:
89
- - - ! '>='
100
+ - - '>='
90
101
  - !ruby/object:Gem::Version
91
102
  version: '0'
92
103
  required_rubygems_version: !ruby/object:Gem::Requirement
93
104
  requirements:
94
- - - ! '>='
105
+ - - '>='
95
106
  - !ruby/object:Gem::Version
96
107
  version: '0'
97
108
  requirements: []
98
- rubyforge_project:
99
- rubygems_version: 2.4.1
100
- signing_key:
109
+ rubyforge_project:
110
+ rubygems_version: 2.4.4
111
+ signing_key:
101
112
  specification_version: 4
102
- summary: The multiline codec will collapse multiline messages and merge them into
103
- a single event.
113
+ summary: The multiline codec will collapse multiline messages and merge them into a single event.
104
114
  test_files:
105
115
  - spec/codecs/multiline_spec.rb
@@ -1,9 +0,0 @@
1
- require "gem_publisher"
2
-
3
- desc "Publish gem to RubyGems.org"
4
- task :publish_gem do |t|
5
- gem_file = Dir.glob(File.expand_path('../*.gemspec',File.dirname(__FILE__))).first
6
- gem = GemPublisher.publish_if_updated(gem_file, :rubygems)
7
- puts "Published #{gem}" if gem
8
- end
9
-
@@ -1,169 +0,0 @@
1
- require "net/http"
2
- require "uri"
3
- require "digest/sha1"
4
-
5
- def vendor(*args)
6
- return File.join("vendor", *args)
7
- end
8
-
9
- directory "vendor/" => ["vendor"] do |task, args|
10
- mkdir task.name
11
- end
12
-
13
- def fetch(url, sha1, output)
14
-
15
- puts "Downloading #{url}"
16
- actual_sha1 = download(url, output)
17
-
18
- if actual_sha1 != sha1
19
- fail "SHA1 does not match (expected '#{sha1}' but got '#{actual_sha1}')"
20
- end
21
- end # def fetch
22
-
23
- def file_fetch(url, sha1)
24
- filename = File.basename( URI(url).path )
25
- output = "vendor/#{filename}"
26
- task output => [ "vendor/" ] do
27
- begin
28
- actual_sha1 = file_sha1(output)
29
- if actual_sha1 != sha1
30
- fetch(url, sha1, output)
31
- end
32
- rescue Errno::ENOENT
33
- fetch(url, sha1, output)
34
- end
35
- end.invoke
36
-
37
- return output
38
- end
39
-
40
- def file_sha1(path)
41
- digest = Digest::SHA1.new
42
- fd = File.new(path, "r")
43
- while true
44
- begin
45
- digest << fd.sysread(16384)
46
- rescue EOFError
47
- break
48
- end
49
- end
50
- return digest.hexdigest
51
- ensure
52
- fd.close if fd
53
- end
54
-
55
- def download(url, output)
56
- uri = URI(url)
57
- digest = Digest::SHA1.new
58
- tmp = "#{output}.tmp"
59
- Net::HTTP.start(uri.host, uri.port, :use_ssl => (uri.scheme == "https")) do |http|
60
- request = Net::HTTP::Get.new(uri.path)
61
- http.request(request) do |response|
62
- fail "HTTP fetch failed for #{url}. #{response}" if [200, 301].include?(response.code)
63
- size = (response["content-length"].to_i || -1).to_f
64
- count = 0
65
- File.open(tmp, "w") do |fd|
66
- response.read_body do |chunk|
67
- fd.write(chunk)
68
- digest << chunk
69
- if size > 0 && $stdout.tty?
70
- count += chunk.bytesize
71
- $stdout.write(sprintf("\r%0.2f%%", count/size * 100))
72
- end
73
- end
74
- end
75
- $stdout.write("\r \r") if $stdout.tty?
76
- end
77
- end
78
-
79
- File.rename(tmp, output)
80
-
81
- return digest.hexdigest
82
- rescue SocketError => e
83
- puts "Failure while downloading #{url}: #{e}"
84
- raise
85
- ensure
86
- File.unlink(tmp) if File.exist?(tmp)
87
- end # def download
88
-
89
- def untar(tarball, &block)
90
- require "archive/tar/minitar"
91
- tgz = Zlib::GzipReader.new(File.open(tarball))
92
- # Pull out typesdb
93
- tar = Archive::Tar::Minitar::Input.open(tgz)
94
- tar.each do |entry|
95
- path = block.call(entry)
96
- next if path.nil?
97
- parent = File.dirname(path)
98
-
99
- mkdir_p parent unless File.directory?(parent)
100
-
101
- # Skip this file if the output file is the same size
102
- if entry.directory?
103
- mkdir path unless File.directory?(path)
104
- else
105
- entry_mode = entry.instance_eval { @mode } & 0777
106
- if File.exists?(path)
107
- stat = File.stat(path)
108
- # TODO(sissel): Submit a patch to archive-tar-minitar upstream to
109
- # expose headers in the entry.
110
- entry_size = entry.instance_eval { @size }
111
- # If file sizes are same, skip writing.
112
- next if stat.size == entry_size && (stat.mode & 0777) == entry_mode
113
- end
114
- puts "Extracting #{entry.full_name} from #{tarball} #{entry_mode.to_s(8)}"
115
- File.open(path, "w") do |fd|
116
- # eof? check lets us skip empty files. Necessary because the API provided by
117
- # Archive::Tar::Minitar::Reader::EntryStream only mostly acts like an
118
- # IO object. Something about empty files in this EntryStream causes
119
- # IO.copy_stream to throw "can't convert nil into String" on JRuby
120
- # TODO(sissel): File a bug about this.
121
- while !entry.eof?
122
- chunk = entry.read(16384)
123
- fd.write(chunk)
124
- end
125
- #IO.copy_stream(entry, fd)
126
- end
127
- File.chmod(entry_mode, path)
128
- end
129
- end
130
- tar.close
131
- File.unlink(tarball) if File.file?(tarball)
132
- end # def untar
133
-
134
- def ungz(file)
135
-
136
- outpath = file.gsub('.gz', '')
137
- tgz = Zlib::GzipReader.new(File.open(file))
138
- begin
139
- File.open(outpath, "w") do |out|
140
- IO::copy_stream(tgz, out)
141
- end
142
- File.unlink(file)
143
- rescue
144
- File.unlink(outpath) if File.file?(outpath)
145
- raise
146
- end
147
- tgz.close
148
- end
149
-
150
- desc "Process any vendor files required for this plugin"
151
- task "vendor" do |task, args|
152
-
153
- @files.each do |file|
154
- download = file_fetch(file['url'], file['sha1'])
155
- if download =~ /.tar.gz/
156
- prefix = download.gsub('.tar.gz', '').gsub('vendor/', '')
157
- untar(download) do |entry|
158
- if !file['files'].nil?
159
- next unless file['files'].include?(entry.full_name.gsub(prefix, ''))
160
- out = entry.full_name.split("/").last
161
- end
162
- File.join('vendor', out)
163
- end
164
- elsif download =~ /.gz/
165
- ungz(download)
166
- end
167
- end
168
-
169
- end