logstash-filter-google_appengine 0.119.0 → 0.120.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 556aca9dd64a5a7d6aac9cd3174d6f59d50d0fb6
-  data.tar.gz: 13b97ac6a47a23669f7ff33eb229624347f88ec0
+  metadata.gz: d611a1687fbd6223a5fddc1369fd0dd87c85a142
+  data.tar.gz: 4d31825089d96b056217accf706425073e019cb7
 SHA512:
-  metadata.gz: 0ea38a0548948593c9af677c9a2b86671ad7dcdc3ed377c47967807b9456efe15b3412486e1c60f2ab724c60d8b039eae7fbc94f493975d25029a29d647a6423
-  data.tar.gz: d5a7a384a5363dd66c3516dc58bec31be25009cff02a229f57a6c50c0918e670e5ac53d1b3890a50e81d192c9578f664acdd9606e965e6163c72950d76055426
+  metadata.gz: f6d922c64b3a9e64a299e4f1dba5621ae3252c5fa2674c1d358a682822a70f85c62c182b692324da837771775e1ae16cf25f22bf7c4f9d254ba6d084a3f662cf
+  data.tar.gz: fc8620ce2f83a84eda9d5a587e844007c5b3e19b1aadd86453e1d5c1ed9f93e0362a7bf00373d2ca511f1a7a2e649701e686901ca8a422384446861e1692203b

data/.gitignore

@@ -84,3 +84,5 @@ com_crashlytics_export_strings.xml
 crashlytics.properties
 crashlytics-build.properties
 
+.mvn
+Gemfile.lock

data/README.md

@@ -20,7 +20,12 @@ Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/log
 ### 1. Plugin Developement and Testing
 
 #### Code
-- To get started, you'll need JRuby with the Bundler gem installed.
+- To get started, you'll need to install [JRuby](http://jruby.org/). Regular Ruby will not work.
+
+- After installing Ruby, install the bundler gem
+```sh
+gem install bundler
+```
 
 - Create a new plugin or clone and existing from the GitHub [logstash-plugins](https://github.com/logstash-plugins) organization. We also provide [example plugins](https://github.com/logstash-plugins?query=example).
 
@@ -40,7 +45,7 @@ bundle install
 - Run tests
 
 ```sh
-bundle exec rspec
+bundle exec rspec spec
 ```
 
 ### 2. Running your unpublished Plugin in Logstash

data/lib/logstash/filters/google_appengine.rb

@@ -21,9 +21,8 @@ class LogStash::Filters::GoogleAppengine < LogStash::Filters::Base
     payload.delete '@type'
     payload['type'] = event.get('type')
     payload['latencyS'] = to_number(payload['latency'])
-    if payload['pendingTime']
-      payload['pendingTimeS'] = to_number(payload['pendingTime'])
-    end
+    payload['pendingTimeS'] = to_number(payload['pendingTime'])
+    payload['appTimeS'] = payload['latencyS'] - payload['pendingTimeS']
     lines = payload.delete 'line'
 
     if lines

data/logstash-filter-google_appengine.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name = 'logstash-filter-google_appengine'
-  s.version = '0.119.0'
+  s.version = '0.120.0'
   s.licenses = ['Apache License (2.0)']
   s.summary = "This filter may be used to decode via inputs appengine logs"
   s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -16,8 +16,10 @@ Gem::Specification.new do |s|
 
   s.metadata = {"logstash_plugin" => "true", "logstash_group" => "filter"}
 
-  s.add_runtime_dependency "logstash-core", '>= 5.0.0', '< 6.0.0'
+  #s.add_runtime_dependency "logstash-core", '>= 5.0.0', '< 6.0.0'
+  s.add_runtime_dependency "logstash-core", '5.0.0'
 
-  s.add_development_dependency 'logstash-devutils', "~> 0"
+  s.add_development_dependency 'logstash-devutils', '>= 1.0.0', '< 2.0.0'
+  s.add_development_dependency 'logstash-core-plugin-api', '2.1.17'
 end
 

data/spec/filters/google_appengine_spec.rb

@@ -14,38 +14,66 @@ describe LogStash::Filters::GoogleAppengine do
   CONFIG
 
   describe "should merge the request payload with the reuest lines data" do
-    test_sample = LogStash::Json::load(File.open("spec/filters/appengine.logs.jsonl", "rb").read)
+    test_sample = LogStash::Json::load(File.open("spec/filters/appengine.logs.json", "rb").read)
     sample (test_sample) do
-      LATENCY_OF_REQUEST = 0.115752
       insist { subject.length } == 3
 
-      insist { subject[0]["message"] }=="IdentityFilter logUserIdentity: [[meta]] <anonymous:true>\n"
-      insist { subject[0]["id"] }== md5.hexdigest(subject[0]["requestId"] + "0")
-      insist { subject[0]["time"] } == "2015-09-03T10:59:40.589Z"
-      insist { subject[0]["position"] } == 0
-      insist { subject[0]["@type"] } == nil
-      insist { subject[0]["latencyS"] } == LATENCY_OF_REQUEST
-      insist { subject[0]["pendingTimeS"] } == nil
-
-      insist { subject[1]["message"] }=="HttpOnlyFilter getSession: add additional Set-Cookie with httpOnly-flag for JSESSIONID\n"
-      insist { subject[1]["id"] } == md5.hexdigest(subject[1]["requestId"] + "1")
-      insist { subject[1]["@type"] } == nil
-      insist { subject[1]["time"] } =="2015-09-03T10:59:40.65Z"
-      insist { subject[1]["position"] } == 1
-      insist { subject[0]["pendingTimeS"] } == nil
+      insist { subject[0].get("message") }=="IdentityFilter logUserIdentity: [[meta]] <anonymous:true>\n"
+      insist { subject[0].get("id") }== md5.hexdigest(subject[0].get("requestId") + "0")
+      insist { subject[0].get("time") } == "2015-09-03T10:59:40.589Z"
+      insist { subject[0].get("position") } == 0
+      insist { subject[0].get("@type") } == nil
+
+      insist { subject[1].get("message") }=="HttpOnlyFilter getSession: add additional Set-Cookie with httpOnly-flag for JSESSIONID\n"
+      insist { subject[1].get("id") } == md5.hexdigest(subject[1].get("requestId") + "1")
+      insist { subject[1].get("@type") } == nil
+      insist { subject[1].get("time") } =="2015-09-03T10:59:40.65Z"
+      insist { subject[1].get("position") } == 1
     end
   end
 
   describe "should handle logs even when they have no lines" do
-    test_sample = LogStash::Json.load(File.open("spec/filters/appengine.logs-without-lines.jsonl", "rb").read)
+    test_sample = LogStash::Json.load(File.open("spec/filters/appengine.logs-without-lines.json", "rb").read)
+    sample (test_sample) do
+
+      insist { subject.get("resource") } == "/images/website/welcome/keyFeatures/objectives.jpg"
+      insist { subject.get("id") } == md5.hexdigest(subject.get("requestId"))
+      insist { subject.get("time") } == subject.get("endTime")
+      insist { subject.get("@type") } == nil
+    end
+  end
+
+  describe "should convert latency (string) to latencyS (number)" do
+    test_sample = LogStash::Json.load(File.open("spec/filters/log-with-pendingTime.json", "rb").read)
+    sample (test_sample) do
+      insist { subject[0].get("latency") } == "0.779603s"
+      insist { subject[0].get("latencyS") } == 0.779603
+    end
+  end
+
+  describe "should convert pendingTime (string) to pendingTimeS (number)" do
+    test_sample = LogStash::Json.load(File.open("spec/filters/log-with-pendingTime.json", "rb").read)
     sample (test_sample) do
+      insist { subject[0].get("pendingTime") } == "0.712152958s"
+      insist { subject[0].get("pendingTimeS") } == 0.712152958
+    end
+  end
 
-      insist { subject["resource"] } == "/images/website/welcome/keyFeatures/objectives.jpg"
-      insist { subject["id"] } == md5.hexdigest(subject["requestId"])
-      insist { subject["time"] } == subject["endTime"]
-      insist { subject["@type"] } == nil
+  describe "should convert missing pendingTime to pendingTimeS is 0" do
+    test_sample = LogStash::Json.load(File.open("spec/filters/log-without-pendingTime.json", "rb").read)
+    sample (test_sample) do
+      insist { subject[0].get("pendingTime") } == nil
+      insist { subject[0].get("pendingTimeS") } == 0
     end
   end
+
+  describe "should calculate appTimeS from latencyS and pendingTimeS" do
+    test_sample = LogStash::Json.load(File.open("spec/filters/log-with-pendingTime.json", "rb").read)
+    sample (test_sample) do
+      insist { subject[0].get("appTimeS") } == 0.779603 - 0.712152958
+    end
+  end
+
 end
 
 

data/spec/filters/log-with-pendingTime.json

@@ -0,0 +1 @@
+{"httpRequest":{"status":200},"insertId":"587e51b60003881392cdafdb","log":"appengine.googleapis.com/request_log","metadata":{"labels":{"appengine.googleapis.com/clone_id":"00c61b117c4f6c766c245383b11b010fd60fe8947ea802c068ad8db2eb45c60880b2ed170e6cb624","appengine.googleapis.com/module_id":"default","appengine.googleapis.com/request_id":"587e51b500ff0688e147d97bf70001737e736d616c6c2d696d70726f76656d656e74732d687264000133000100","appengine.googleapis.com/version_id":"3","clone_id":"00c61b117c4f6c766c245383b11b010fd60fe8947ea802c068ad8db2eb45c60880b2ed170e6cb624","module_id":"default","request_id":"587e51b500ff0688e147d97bf70001737e736d616c6c2d696d70726f76656d656e74732d687264000133000100","version_id":"3"},"projectId":"small-improvements-hrd","serviceName":"appengine.googleapis.com","severity":"INFO","timestamp":"2017-01-17T17:17:41.428257Z","zone":"us2"},"operation":{"first":true,"id":"587e51b500ff0688e147d97bf70001737e736d616c6c2d696d70726f76656d656e74732d687264000133000100","last":true,"producer":"appengine.googleapis.com/request_id"},"protoPayload":{"@type":"type.googleapis.com/google.appengine.logging.v1.RequestLog","appEngineRelease":"1.9.48","appId":"s~small-improvements-hrd","cost":6.0908000000000005e-08,"endTime":"2017-01-17T17:17:42.20786Z","finished":true,"first":true,"host":"customer.small-improvements.com","httpVersion":"HTTP/1.1","instanceId":"00c61b117c4f6c766c245383b11b010fd60fe8947ea802c068ad8db2eb45c60880b2ed170e6cb624","instanceIndex":-1,"ip":"65.25.254.214","latency":"0.779603s","line":[{"logMessage":"IdentityContextLogger logUserAndHisProperties: [Identity][BEGIN] userId:*tf83mvBtXCwmMpvkyrvTS userMail:john.doe@customer.com companyId:inpfVlYp1rug1ndpA7r7TV isSI:false\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-01-17T17:17:42.195Z"},{"logMessage":"IdentityContextLogger logContextEnd: [Identity END]\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-01-17T17:17:42.206Z"}],"megaCycles":"39","method":"GET","pendingTime":"0.712152958s","requestId":"587e51b500ff0688e147d97bf70001737e736d616c6c2d696d70726f76656d656e74732d687264000133000100","resource":"/api/avatars/2/DT.svg?size=50","responseSize":"545","startTime":"2017-01-17T17:17:41.428257Z","status":200,"urlMapEntry":"unused","userAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.95 Safari/537.36","versionId":"3"}}

data/spec/filters/log-without-pendingTime.json

@@ -0,0 +1 @@
+{"httpRequest":{"status":200},"insertId":"587e4d90000cdd3a0ec11b34","log":"appengine.googleapis.com/request_log","metadata":{"labels":{"appengine.googleapis.com/clone_id":"00c61b117cf26d77f54f5c83731f597b2b82d3856b7706be25b95a1657b7fc4903828a568855b3f7","appengine.googleapis.com/module_id":"default","appengine.googleapis.com/request_id":"587e4d9000ff03f6c7ed2936820001737e736d616c6c2d696d70726f76656d656e74732d687264000133000100","appengine.googleapis.com/version_id":"3","clone_id":"00c61b117cf26d77f54f5c83731f597b2b82d3856b7706be25b95a1657b7fc4903828a568855b3f7","module_id":"default","request_id":"587e4d9000ff03f6c7ed2936820001737e736d616c6c2d696d70726f76656d656e74732d687264000133000100","version_id":"3"},"projectId":"small-improvements-hrd","serviceName":"appengine.googleapis.com","severity":"INFO","timestamp":"2017-01-17T17:00:00.259783Z","zone":"us2"},"operation":{"first":true,"id":"587e4d9000ff03f6c7ed2936820001737e736d616c6c2d696d70726f76656d656e74732d687264000133000100","last":true,"producer":"appengine.googleapis.com/request_id"},"protoPayload":{"@type":"type.googleapis.com/google.appengine.logging.v1.RequestLog","appEngineRelease":"1.9.48","appId":"s~small-improvements-hrd","cost":1.3321599999999999e-07,"endTime":"2017-01-17T17:00:00.421235Z","finished":true,"first":true,"host":"customer.small-improvements.com","httpVersion":"HTTP/1.1","instanceId":"00c61b117cf26d77f54f5c83731f597b2b82d3856b7706be25b95a1657b7fc4903828a568855b3f7","instanceIndex":-1,"ip":"209.152.64.69","latency":"0.161452s","line":[{"logMessage":"IdentityContextLogger logUserAndHisProperties: [Identity][BEGIN] userId:7LudpFWKJAP9Ze3kixSICQ userMail:alice@customer.com companyId:ZjxohdxgW3w6yfLXibPTq6 isSI:false\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-01-17T17:00:00.289Z"},{"logMessage":"MetricsFlushingFilter acquireLockAndSendMetrics: Flush interval of PT10S exceeded sending metrics now\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-01-17T17:00:00.299Z"},{"logMessage":"IdentityContextLogger logContextEnd: [Identity END]\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-01-17T17:00:00.419Z"}],"megaCycles":"57","method":"GET","referrer":"https://customer.small-improvements.com/","requestId":"587e4d9000ff03f6c7ed2936820001737e736d616c6c2d696d70726f76656d656e74732d687264000133000100","resource":"/api/avatars/23/AR.svg","responseSize":"550","startTime":"2017-01-17T17:00:00.259783Z","status":200,"traceId":"cde94892157fbf26b5144f6db20b34a8","urlMapEntry":"unused","userAgent":"Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; TNJB; rv:11.0) like Gecko","versionId":"3"}}

metadata

@@ -1,47 +1,61 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-google_appengine
 version: !ruby/object:Gem::Version
-  version: 0.119.0
+  version: 0.120.0
 platform: ruby
 authors:
 - Small Improvements
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-10-28 00:00:00.000000000 Z
+date: 2017-01-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: logstash-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 5.0.0
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - '='
       - !ruby/object:Gem::Version
         version: 5.0.0
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.0.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 6.0.0
-  requirement: !ruby/object:Gem::Requirement
+        version: 2.0.0
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 5.0.0
+        version: 1.0.0
     - - "<"
       - !ruby/object:Gem::Version
-        version: 6.0.0
+        version: 2.0.0
   prerelease: false
-  type: :runtime
+  type: :development
 - !ruby/object:Gem::Dependency
-  name: logstash-devutils
-  version_requirements: !ruby/object:Gem::Requirement
+  name: logstash-core-plugin-api
+  requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
-  requirement: !ruby/object:Gem::Requirement
+        version: 2.1.17
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.1.17
   prerelease: false
   type: :development
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
@@ -57,9 +71,11 @@ files:
 - Rakefile
 - lib/logstash/filters/google_appengine.rb
 - logstash-filter-google_appengine.gemspec
-- spec/filters/appengine.logs-without-lines.jsonl
-- spec/filters/appengine.logs.jsonl
+- spec/filters/appengine.logs-without-lines.json
+- spec/filters/appengine.logs.json
 - spec/filters/google_appengine_spec.rb
+- spec/filters/log-with-pendingTime.json
+- spec/filters/log-without-pendingTime.json
 homepage: https://github.com/SmallImprovements/logstash-filter-google_appengine
 licenses:
 - Apache License (2.0)
@@ -82,11 +98,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project:
-rubygems_version: 2.4.8
+rubygems_version: 2.6.8
 signing_key:
 specification_version: 4
 summary: This filter may be used to decode via inputs appengine logs
 test_files:
-- spec/filters/appengine.logs-without-lines.jsonl
-- spec/filters/appengine.logs.jsonl
+- spec/filters/appengine.logs-without-lines.json
+- spec/filters/appengine.logs.json
 - spec/filters/google_appengine_spec.rb
+- spec/filters/log-with-pendingTime.json
+- spec/filters/log-without-pendingTime.json