logstash-codec-google_appengine 1.2.0 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 9ea3583c470ffc16060fd747cd7356cf06409623
-  data.tar.gz: deb063973fa616655f86a96f02d5300f037dca15
+  metadata.gz: 65a25d37344d2d857ced131dfbca80891278dd78
+  data.tar.gz: 7b0c1042da8f3905e01f7743cc1868b1c1e4bbcb
 SHA512:
-  metadata.gz: 23b6dadcb11cdb0e6114f5bde7f871e2893b4018f540e58f727c3bd8af852c27407cc9d11b115ca6b84d67e55a9de01bae0d2782762a45edc2d72b1f7a85e4c5
-  data.tar.gz: 3c5eb1ca67af2bd978d695b98cfd8d6ed33627f96ddde6387feec451eb5a47ca1db1a0a32809d04ca0d61905a03af998ff3e1e2506abd5ca8bebbb1e1f8e445f
+  metadata.gz: 70f2f14571103eb44329f903f97b9e2e1483ecbca36954505b7b384f3f7ca04483ee7d1ca5b5208cb982390686b1ede49ff9f0f84010dc84894fbfbfcf3d73ac
+  data.tar.gz: 19308f459a967b1d2f7630f8b8f3ad6c633626e519384a969894b08da7991bc6422074d361412a332edac98da981c1662881dab78059ebfb4412674be474f3db

data/lib/logstash/codecs/google_appengine.rb

@@ -22,14 +22,13 @@ class LogStash::Codecs::GoogleAppengine < LogStash::Codecs::Base
           yield LogStash::Event.new(flattenedJson)
         }
       end
-      rescue => e
-        @logger.info("Failed to process data", :error => e, :data => data)
-        yield LogStash::Event.new("message" => data, "tags" => ["_googleappengineparsefailure"])
+    rescue => e
+      @logger.info("Failed to process data", :error => e, :data => data)
+      yield LogStash::Event.new("message" => data, "tags" => ["_googleappengineparsefailure"])
     end
   end
 end
 
-
 private
 
 def is_parse_failure(event)
@@ -39,6 +38,10 @@ end
 def flatten(event)
   payload = event['protoPayload']
   lines = payload['line']
-  payload.delete('line')
-  lines.map { |line| payload.merge(line) }
+  if lines
+    payload.delete('line')
+    lines.map { |line| payload.merge(line) }
+  else
+    [payload]
+  end
 end

data/logstash-codec-google_appengine.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name = 'logstash-codec-google_appengine'
-  s.version = '1.2.0'
+  s.version = '1.3.0'
   s.licenses = ['Apache License (2.0)']
   s.summary = "This codec may be used to decode via inputs appengine logs"
   s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"

data/spec/codecs/appengine.logs-without-lines.jsonl

@@ -0,0 +1 @@
+{"metadata":{"serviceName":"appengine.googleapis.com","zone":"us9","labels":{"appengine.googleapis.com/module_id":"default","appengine.googleapis.com/request_id":"55e80a9c00ff096e0f91589f3f0001737e73686f7070696e672d67616d652d68726400013100010103","appengine.googleapis.com/version_id":"1"},"timestamp":"2015-09-03T08:53:48.617999Z"},"protoPayload":{"@type":"type.googleapis.com/google.appengine.logging.v1.RequestLog","appId":"s~shopping-game-hrd","versionId":"1","requestId":"NTVlODBhOWMwMGZmMDk2ZTBmOTE1ODlmM2YwMDAxNzM3ZTczNjg2ZjcwNzA2OTZlNjcyZDY3NjE2ZDY1MmQ2ODcyNjQwMDAxMzEwMDAxMDEwMw==","ip":"78.55.216.42","startTime":"2015-09-03T08:53:48.617999Z","endTime":"2015-09-03T08:53:48.671351Z","latency":"0.053352s","method":"GET","resource":"/images/website/welcome/keyFeatures/objectives.jpg","httpVersion":"HTTP/1.1","status":200,"userAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2439.0 Safari/537.36","host":"shopping-game.small-improvements.com","instanceIndex":-1,"appEngineRelease":"1.9.25"},"insertId":"2015-09-03|01:53:50.387721-07|10.106.166.136|1292431933","log":"appengine.googleapis.com/request_log","httpRequest":{"status":200}}

data/spec/codecs/google_appengine_spec.rb

@@ -9,30 +9,48 @@ describe LogStash::Codecs::GoogleAppengine do
     next LogStash::Codecs::GoogleAppengine.new
   end
 
-  data = File.open("spec/codecs/appengine.logs.jsonl", "rb").read
-
-  context "#decode" do
-    it "should return an event from json data" do
-      subject.decode(data) do |event|
-        insist { event.is_a? LogStash::Event }
-        insist { event["@type"] } == "type.googleapis.com/google.appengine.logging.v1.RequestLog"
-      end
+
+
+context "#decode" do
+
+  it "should return an event from json data" do
+    data = File.open("spec/codecs/appengine.logs.jsonl", "rb").read
+    subject.decode(data) do |event|
+      insist { event.is_a? LogStash::Event }
+      insist { event["@type"] } == "type.googleapis.com/google.appengine.logging.v1.RequestLog"
     end
+  end
+
+  it "should merge the request payload with the reuest lines data" do
+    data = File.open("spec/codecs/appengine.logs.jsonl", "rb").read
+    collector = Array.new
+    subject.decode(data) do |event|
+      collector.push(event)
+    end
+
+    expect(collector.size).to eq(3)
 
-    it "should merge the request payload with the reuest lines data" do
-      collector = Array.new
-      subject.decode(data) do |event|
-        collector.push(event)
-      end
+    expect(collector[0]["@type"]).to eq("type.googleapis.com/google.appengine.logging.v1.RequestLog")
+    expect(collector[0]["logMessage"]).to eq("IdentityFilter logUserIdentity: [[meta]] <anonymous:true>\n")
 
-      expect(collector.size).to eq(3)
+    expect(collector[1]["@type"]).to eq("type.googleapis.com/google.appengine.logging.v1.RequestLog")
+    expect(collector[1]["logMessage"]).to eq("HttpOnlyFilter getSession: add additional Set-Cookie with httpOnly-flag for JSESSIONID\n")
+  end
+
+  it "should handle logs even when they have no lines" do
 
-      expect(collector[0]["@type"]).to eq("type.googleapis.com/google.appengine.logging.v1.RequestLog")
-      expect(collector[0]["logMessage"]).to eq("IdentityFilter logUserIdentity: [[meta]] <anonymous:true>\n")
+    data = File.open("spec/codecs/appengine.logs-without-lines.jsonl", "rb").read
 
-      expect(collector[1]["@type"]).to eq("type.googleapis.com/google.appengine.logging.v1.RequestLog")
-      expect(collector[1]["logMessage"]).to eq("HttpOnlyFilter getSession: add additional Set-Cookie with httpOnly-flag for JSESSIONID\n")
+    collector = Array.new
+
+    subject.decode(data) do |event|
+      collector.push(event)
     end
+
+    expect(collector.size).to eq(1)
+
+    expect(collector[0]["@type"]).to eq("type.googleapis.com/google.appengine.logging.v1.RequestLog")
+    expect(collector[0]["resource"]).to eq("/images/website/welcome/keyFeatures/objectives.jpg")
   end
 
   it "falls back to plain text" do
@@ -46,3 +64,6 @@ describe LogStash::Codecs::GoogleAppengine do
     insist { decoded } == true
   end
 end
+
+
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-codec-google_appengine
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.0
 platform: ruby
 authors:
 - Small Improvements
@@ -74,6 +74,7 @@ files:
 - lib/logstash/codecs/google_appengine.rb
 - logstash-codec-google-appengine.iml
 - logstash-codec-google_appengine.gemspec
+- spec/codecs/appengine.logs-without-lines.jsonl
 - spec/codecs/appengine.logs.jsonl
 - spec/codecs/google_appengine_spec.rb
 homepage: https://github.com/MichaelRuhwedel/logstash-codec-google-appengine
@@ -103,5 +104,6 @@ signing_key:
 specification_version: 4
 summary: This codec may be used to decode via inputs appengine logs
 test_files:
+- spec/codecs/appengine.logs-without-lines.jsonl
 - spec/codecs/appengine.logs.jsonl
 - spec/codecs/google_appengine_spec.rb