logstash-codec-oldlogstashjson 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml ADDED
@@ -0,0 +1,15 @@
1
+ ---
2
+ !binary "U0hBMQ==":
3
+ metadata.gz: !binary |-
4
+ YzA5MDVkOTM1YmI1N2ZlOGNkNGYzMThjN2UxMWY4M2FhZGExNTJkZg==
5
+ data.tar.gz: !binary |-
6
+ ZWIzNTc2OWQ2ZDA4MzRmNTFiNTA2NGE2NzU3MGYxMGE0ODNiODczYQ==
7
+ SHA512:
8
+ metadata.gz: !binary |-
9
+ MjM2MmYzMmQzNTZiNDJiZDk5MmQ3ODI0NWY4ZjkzNDk0YTk0ZWM4ZGE4OWJm
10
+ ZTZmMGJhY2M4NDQ3MTI4ZmUxMTJiN2M0MmJlMGI4NmRlNGUyYmM0MGQyNjc3
11
+ ZDFmZWYyNGMzNTU1Mzg4OTQyMzM3NDAyOTlkZTc0ZmY1YzkwODg=
12
+ data.tar.gz: !binary |-
13
+ ODViODIwZjZlMzY5NmJhOGQxNTVmOWE2N2ZmNWI4YTRmMzYwZWJjZTUyNGI5
14
+ YmVkMTBmOWM3MGMyY2ZjMDI1ZmRjYWUzZTU4YmZjZGQwMWE3YjdmMGVlOTJi
15
+ YjFjZjcwNDQ2NjIyNGFjNzgzMDRlOWQ0MTEwNDI5MzU5MjA0ZTU=
data/.gitignore ADDED
@@ -0,0 +1,3 @@
1
+ *.gem
2
+ Gemfile.lock
3
+ .bundle
data/Gemfile ADDED
@@ -0,0 +1,4 @@
1
+ source 'http://rubygems.org'
2
+ gem 'rake'
3
+ gem 'gem_publisher'
4
+ gem 'archive-tar-minitar'
data/Rakefile ADDED
@@ -0,0 +1,6 @@
1
+ @files=[]
2
+
3
+ task :default do
4
+ system("rake -T")
5
+ end
6
+
@@ -0,0 +1,57 @@
1
+ # encoding: utf-8
2
+ require "logstash/codecs/base"
3
+ require "logstash/json"
4
+
5
+ class LogStash::Codecs::OldLogStashJSON < LogStash::Codecs::Base
6
+ config_name "oldlogstashjson"
7
+ milestone 2
8
+
9
+ # Map from v0 name to v1 name.
10
+ # Note: @source is gone and has no similar field.
11
+ V0_TO_V1 = {"@timestamp" => "@timestamp", "@message" => "message",
12
+ "@tags" => "tags", "@type" => "type",
13
+ "@source_host" => "host", "@source_path" => "path"}
14
+
15
+ public
16
+ def decode(data)
17
+ begin
18
+ obj = LogStash::Json.load(data.force_encoding(Encoding::UTF_8))
19
+ rescue LogStash::Json::ParserError => e
20
+ @logger.info("JSON parse failure. Falling back to plain-text", :error => e, :data => data)
21
+ yield LogStash::Event.new("message" => data)
22
+ return
23
+ end
24
+
25
+ h = {}
26
+
27
+ # Convert the old logstash schema to the new one.
28
+ V0_TO_V1.each do |key, val|
29
+ h[val] = obj[key] if obj.include?(key)
30
+ end
31
+
32
+ h.merge!(obj["@fields"]) if obj["@fields"].is_a?(Hash)
33
+ yield LogStash::Event.new(h)
34
+ end # def decode
35
+
36
+ public
37
+ def encode(event)
38
+ h = {}
39
+
40
+ # Convert the new logstash schema to the old one.
41
+ V0_TO_V1.each do |key, val|
42
+ h[key] = event[val] if event.include?(val)
43
+ end
44
+
45
+ event.to_hash.each do |field, val|
46
+ # TODO: might be better to V1_TO_V0 = V0_TO_V1.invert during
47
+ # initialization than V0_TO_V1.has_value? within loop
48
+ next if field == "@version" or V0_TO_V1.has_value?(field)
49
+ h["@fields"] ||= {}
50
+ h["@fields"][field] = val
51
+ end
52
+
53
+ # Tack on a \n because JSON outputs 1.1.x had them.
54
+ @on_event.call(LogStash::Json.dump(h) + NL)
55
+ end # def encode
56
+
57
+ end # class LogStash::Codecs::OldLogStashJSON
@@ -0,0 +1,26 @@
1
+ Gem::Specification.new do |s|
2
+
3
+ s.name = 'logstash-codec-oldlogstashjson'
4
+ s.version = '0.1.0'
5
+ s.licenses = ['Apache License (2.0)']
6
+ s.summary = "Codec to convert v0 logstash format to v1"
7
+ s.description = "Codec to convert v0 logstash format to v1"
8
+ s.authors = ["Elasticsearch"]
9
+ s.email = 'richard.pijnenburg@elasticsearch.com'
10
+ s.homepage = "http://logstash.net/"
11
+ s.require_paths = ["lib"]
12
+
13
+ # Files
14
+ s.files = `git ls-files`.split($\)
15
+
16
+ # Tests
17
+ s.test_files = s.files.grep(%r{^(test|spec|features)/})
18
+
19
+ # Special flag to let us know this is actually a logstash plugin
20
+ s.metadata = { "logstash_plugin" => "true", "group" => "codec" }
21
+
22
+ # Gem dependencies
23
+ s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
24
+
25
+ end
26
+
@@ -0,0 +1,9 @@
1
+ require "gem_publisher"
2
+
3
+ desc "Publish gem to RubyGems.org"
4
+ task :publish_gem do |t|
5
+ gem_file = Dir.glob(File.expand_path('../*.gemspec',File.dirname(__FILE__))).first
6
+ gem = GemPublisher.publish_if_updated(gem_file, :rubygems)
7
+ puts "Published #{gem}" if gem
8
+ end
9
+
@@ -0,0 +1,169 @@
1
+ require "net/http"
2
+ require "uri"
3
+ require "digest/sha1"
4
+
5
+ def vendor(*args)
6
+ return File.join("vendor", *args)
7
+ end
8
+
9
+ directory "vendor/" => ["vendor"] do |task, args|
10
+ mkdir task.name
11
+ end
12
+
13
+ def fetch(url, sha1, output)
14
+
15
+ puts "Downloading #{url}"
16
+ actual_sha1 = download(url, output)
17
+
18
+ if actual_sha1 != sha1
19
+ fail "SHA1 does not match (expected '#{sha1}' but got '#{actual_sha1}')"
20
+ end
21
+ end # def fetch
22
+
23
+ def file_fetch(url, sha1)
24
+ filename = File.basename( URI(url).path )
25
+ output = "vendor/#{filename}"
26
+ task output => [ "vendor/" ] do
27
+ begin
28
+ actual_sha1 = file_sha1(output)
29
+ if actual_sha1 != sha1
30
+ fetch(url, sha1, output)
31
+ end
32
+ rescue Errno::ENOENT
33
+ fetch(url, sha1, output)
34
+ end
35
+ end.invoke
36
+
37
+ return output
38
+ end
39
+
40
+ def file_sha1(path)
41
+ digest = Digest::SHA1.new
42
+ fd = File.new(path, "r")
43
+ while true
44
+ begin
45
+ digest << fd.sysread(16384)
46
+ rescue EOFError
47
+ break
48
+ end
49
+ end
50
+ return digest.hexdigest
51
+ ensure
52
+ fd.close if fd
53
+ end
54
+
55
+ def download(url, output)
56
+ uri = URI(url)
57
+ digest = Digest::SHA1.new
58
+ tmp = "#{output}.tmp"
59
+ Net::HTTP.start(uri.host, uri.port, :use_ssl => (uri.scheme == "https")) do |http|
60
+ request = Net::HTTP::Get.new(uri.path)
61
+ http.request(request) do |response|
62
+ fail "HTTP fetch failed for #{url}. #{response}" if [200, 301].include?(response.code)
63
+ size = (response["content-length"].to_i || -1).to_f
64
+ count = 0
65
+ File.open(tmp, "w") do |fd|
66
+ response.read_body do |chunk|
67
+ fd.write(chunk)
68
+ digest << chunk
69
+ if size > 0 && $stdout.tty?
70
+ count += chunk.bytesize
71
+ $stdout.write(sprintf("\r%0.2f%%", count/size * 100))
72
+ end
73
+ end
74
+ end
75
+ $stdout.write("\r \r") if $stdout.tty?
76
+ end
77
+ end
78
+
79
+ File.rename(tmp, output)
80
+
81
+ return digest.hexdigest
82
+ rescue SocketError => e
83
+ puts "Failure while downloading #{url}: #{e}"
84
+ raise
85
+ ensure
86
+ File.unlink(tmp) if File.exist?(tmp)
87
+ end # def download
88
+
89
+ def untar(tarball, &block)
90
+ require "archive/tar/minitar"
91
+ tgz = Zlib::GzipReader.new(File.open(tarball))
92
+ # Pull out typesdb
93
+ tar = Archive::Tar::Minitar::Input.open(tgz)
94
+ tar.each do |entry|
95
+ path = block.call(entry)
96
+ next if path.nil?
97
+ parent = File.dirname(path)
98
+
99
+ mkdir_p parent unless File.directory?(parent)
100
+
101
+ # Skip this file if the output file is the same size
102
+ if entry.directory?
103
+ mkdir path unless File.directory?(path)
104
+ else
105
+ entry_mode = entry.instance_eval { @mode } & 0777
106
+ if File.exists?(path)
107
+ stat = File.stat(path)
108
+ # TODO(sissel): Submit a patch to archive-tar-minitar upstream to
109
+ # expose headers in the entry.
110
+ entry_size = entry.instance_eval { @size }
111
+ # If file sizes are same, skip writing.
112
+ next if stat.size == entry_size && (stat.mode & 0777) == entry_mode
113
+ end
114
+ puts "Extracting #{entry.full_name} from #{tarball} #{entry_mode.to_s(8)}"
115
+ File.open(path, "w") do |fd|
116
+ # eof? check lets us skip empty files. Necessary because the API provided by
117
+ # Archive::Tar::Minitar::Reader::EntryStream only mostly acts like an
118
+ # IO object. Something about empty files in this EntryStream causes
119
+ # IO.copy_stream to throw "can't convert nil into String" on JRuby
120
+ # TODO(sissel): File a bug about this.
121
+ while !entry.eof?
122
+ chunk = entry.read(16384)
123
+ fd.write(chunk)
124
+ end
125
+ #IO.copy_stream(entry, fd)
126
+ end
127
+ File.chmod(entry_mode, path)
128
+ end
129
+ end
130
+ tar.close
131
+ File.unlink(tarball) if File.file?(tarball)
132
+ end # def untar
133
+
134
+ def ungz(file)
135
+
136
+ outpath = file.gsub('.gz', '')
137
+ tgz = Zlib::GzipReader.new(File.open(file))
138
+ begin
139
+ File.open(outpath, "w") do |out|
140
+ IO::copy_stream(tgz, out)
141
+ end
142
+ File.unlink(file)
143
+ rescue
144
+ File.unlink(outpath) if File.file?(outpath)
145
+ raise
146
+ end
147
+ tgz.close
148
+ end
149
+
150
+ desc "Process any vendor files required for this plugin"
151
+ task "vendor" do |task, args|
152
+
153
+ @files.each do |file|
154
+ download = file_fetch(file['url'], file['sha1'])
155
+ if download =~ /.tar.gz/
156
+ prefix = download.gsub('.tar.gz', '').gsub('vendor/', '')
157
+ untar(download) do |entry|
158
+ if !file['files'].nil?
159
+ next unless file['files'].include?(entry.full_name.gsub(prefix, ''))
160
+ out = entry.full_name.split("/").last
161
+ end
162
+ File.join('vendor', out)
163
+ end
164
+ elsif download =~ /.gz/
165
+ ungz(download)
166
+ end
167
+ end
168
+
169
+ end
@@ -0,0 +1,56 @@
1
+ require "logstash/codecs/oldlogstashjson"
2
+ require "logstash/event"
3
+ require "logstash/json"
4
+ require "insist"
5
+
6
+ describe LogStash::Codecs::OldLogStashJSON do
7
+ subject do
8
+ next LogStash::Codecs::OldLogStashJSON.new
9
+ end
10
+
11
+ context "#decode" do
12
+ it "should return a new (v1) event from old (v0) json data" do
13
+ data = {"@message" => "bar", "@source_host" => "localhost",
14
+ "@tags" => ["a","b","c"]}
15
+ subject.decode(LogStash::Json.dump(data)) do |event|
16
+ insist { event.is_a? LogStash::Event }
17
+ insist { event["@timestamp"] } != nil
18
+ insist { event["type"] } == data["@type"]
19
+ insist { event["message"] } == data["@message"]
20
+ insist { event["host"] } == data["@source_host"]
21
+ insist { event["tags"] } == data["@tags"]
22
+ insist { event["path"] } == nil # @source_path not in v0 test data
23
+ end
24
+ end
25
+
26
+ it "should accept invalid json" do
27
+ subject.decode("some plain text") do |event|
28
+ insist { event["message"] } == "some plain text"
29
+ end
30
+ end
31
+ end
32
+
33
+ context "#encode" do
34
+ it "should return old (v0) json data" do
35
+ data = {"type" => "t", "message" => "wat!?",
36
+ "host" => "localhost", "path" => "/foo",
37
+ "tags" => ["a","b","c"],
38
+ "bah" => "baz"}
39
+ event = LogStash::Event.new(data)
40
+ got_event = false
41
+ subject.on_event do |d|
42
+ insist { LogStash::Json.load(d)["@timestamp"] } != nil
43
+ insist { LogStash::Json.load(d)["@type"] } == data["type"]
44
+ insist { LogStash::Json.load(d)["@message"] } == data["message"]
45
+ insist { LogStash::Json.load(d)["@source_host"] } == data["host"]
46
+ insist { LogStash::Json.load(d)["@source_path"] } == data["path"]
47
+ insist { LogStash::Json.load(d)["@tags"] } == data["tags"]
48
+ insist { LogStash::Json.load(d)["@fields"]["bah"] } == "baz"
49
+ insist { LogStash::Json.load(d)["@fields"]["@version"] } == nil
50
+ got_event = true
51
+ end
52
+ subject.encode(event)
53
+ insist { got_event }
54
+ end
55
+ end
56
+ end
metadata ADDED
@@ -0,0 +1,74 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: logstash-codec-oldlogstashjson
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.0
5
+ platform: ruby
6
+ authors:
7
+ - Elasticsearch
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2014-11-02 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: logstash
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - ! '>='
18
+ - !ruby/object:Gem::Version
19
+ version: 1.4.0
20
+ - - <
21
+ - !ruby/object:Gem::Version
22
+ version: 2.0.0
23
+ type: :runtime
24
+ prerelease: false
25
+ version_requirements: !ruby/object:Gem::Requirement
26
+ requirements:
27
+ - - ! '>='
28
+ - !ruby/object:Gem::Version
29
+ version: 1.4.0
30
+ - - <
31
+ - !ruby/object:Gem::Version
32
+ version: 2.0.0
33
+ description: Codec to convert v0 logstash format to v1
34
+ email: richard.pijnenburg@elasticsearch.com
35
+ executables: []
36
+ extensions: []
37
+ extra_rdoc_files: []
38
+ files:
39
+ - .gitignore
40
+ - Gemfile
41
+ - Rakefile
42
+ - lib/logstash/codecs/oldlogstashjson.rb
43
+ - logstash-codec-oldlogstashjson.gemspec
44
+ - rakelib/publish.rake
45
+ - rakelib/vendor.rake
46
+ - spec/codecs/oldlogstashjson_spec.rb
47
+ homepage: http://logstash.net/
48
+ licenses:
49
+ - Apache License (2.0)
50
+ metadata:
51
+ logstash_plugin: 'true'
52
+ group: codec
53
+ post_install_message:
54
+ rdoc_options: []
55
+ require_paths:
56
+ - lib
57
+ required_ruby_version: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - ! '>='
60
+ - !ruby/object:Gem::Version
61
+ version: '0'
62
+ required_rubygems_version: !ruby/object:Gem::Requirement
63
+ requirements:
64
+ - - ! '>='
65
+ - !ruby/object:Gem::Version
66
+ version: '0'
67
+ requirements: []
68
+ rubyforge_project:
69
+ rubygems_version: 2.4.1
70
+ signing_key:
71
+ specification_version: 4
72
+ summary: Codec to convert v0 logstash format to v1
73
+ test_files:
74
+ - spec/codecs/oldlogstashjson_spec.rb