logstash-filter-google_appengine 0.120.0 → 0.121.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: d611a1687fbd6223a5fddc1369fd0dd87c85a142
-  data.tar.gz: 4d31825089d96b056217accf706425073e019cb7
+  metadata.gz: a3b95e78210a10d77a1042ed9d7a598bcd9d577a
+  data.tar.gz: 83eee1233cf84a75374c1f64984c3ef9b8ecfa71
 SHA512:
-  metadata.gz: f6d922c64b3a9e64a299e4f1dba5621ae3252c5fa2674c1d358a682822a70f85c62c182b692324da837771775e1ae16cf25f22bf7c4f9d254ba6d084a3f662cf
-  data.tar.gz: fc8620ce2f83a84eda9d5a587e844007c5b3e19b1aadd86453e1d5c1ed9f93e0362a7bf00373d2ca511f1a7a2e649701e686901ca8a422384446861e1692203b
+  metadata.gz: 3675bf66f7ff2b80afc9c7303a51e777a244bfbeb2b435f0dce665cec426319ce9a40934c8055b2bf76d72e01d2060369aa2342759c764b09e74f67413adf018
+  data.tar.gz: b625103c5261ecaecae991d48c55ac29bde8143f859f492febd6cb63f039303a503bf3ce6b521feb480a8a8b881c522583bae6db8c53c388962ea45a59bcb624

data/lib/logstash/filters/google_appengine.rb

@@ -1,16 +1,14 @@
 # encoding: utf-8
 require "logstash/filters/base"
 require "logstash/namespace"
-require 'digest'
 
 class LogStash::Filters::GoogleAppengine < LogStash::Filters::Base
 
   config_name "google_appengine"
 
   public
+
   def register
-    @md5 = Digest::MD5.new
-    @semaphore = Mutex.new
   end
 
   def filter(event)
@@ -43,7 +41,7 @@ class LogStash::Filters::GoogleAppengine < LogStash::Filters::Base
 
   def collect_line_data(i, line, payload)
     {
-        'id' => get_id(payload['requestId'] + i.to_s),
+        'id' => (payload['requestId'] + i.to_s),
         'message' => line.delete('logMessage'),
         'position' => i
     }
@@ -54,7 +52,7 @@ class LogStash::Filters::GoogleAppengine < LogStash::Filters::Base
   # noinspection RubyStringKeysInHashInspection
   def collect_resource_request_data(payload)
     {
-        'id' => get_id(payload['requestId']),
+        'id' => payload['requestId'],
         'time' => payload['endTime'],
         'position' => 0
     }
@@ -67,10 +65,4 @@ class LogStash::Filters::GoogleAppengine < LogStash::Filters::Base
     new_event
   end
 
-  def get_id(source)
-    @semaphore.synchronize {
-      @md5.hexdigest(source) #md5 isn't threadsafe :(
-    }
-  end
-
 end # class LogStash::Filters::GoogleAppengine

data/logstash-filter-google_appengine.gemspec

@@ -1,7 +1,7 @@
 Gem::Specification.new do |s|
 
   s.name = 'logstash-filter-google_appengine'
-  s.version = '0.120.0'
+  s.version = '0.121.0'
   s.licenses = ['Apache License (2.0)']
   s.summary = "This filter may be used to decode via inputs appengine logs"
   s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -16,10 +16,9 @@ Gem::Specification.new do |s|
 
   s.metadata = {"logstash_plugin" => "true", "logstash_group" => "filter"}
 
-  #s.add_runtime_dependency "logstash-core", '>= 5.0.0', '< 6.0.0'
-  s.add_runtime_dependency "logstash-core", '5.0.0'
+  s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+
+  s.add_development_dependency 'logstash-devutils'
 
-  s.add_development_dependency 'logstash-devutils', '>= 1.0.0', '< 2.0.0'
-  s.add_development_dependency 'logstash-core-plugin-api', '2.1.17'
 end
 

data/spec/filters/appengine.logs-v2.json

@@ -0,0 +1 @@
+{"httpRequest":{"status":200},"insertId":"58aa3f900007437ae2d40d60","labels":{"clone_id":"00c61b117cfd455a65e9c255fe7ad6e20ee1f8a53bd8decf33c2a74b062edf2f4d9a62eec424"},"logName":"projects/small-improvements-hrd/logs/appengine.googleapis.com%2Frequest_log","operation":{"first":true,"id":"58aa3f8f00ff03c6042eabdadf0001737e736d616c6c2d696d70726f76656d656e74732d687264000133000100","last":true,"producer":"appengine.googleapis.com/request_id"},"protoPayload":{"@type":"type.googleapis.com/google.appengine.logging.v1.RequestLog","appEngineRelease":"1.9.48","appId":"s~small-improvements-hrd","cost":1.12093e-07,"endTime":"2017-02-20T00:59:59.659992Z","finished":true,"first":true,"host":"radius.small-improvements.com","httpVersion":"HTTP/1.1","instanceId":"00c61b117cfd455a65e9c255fe7ad6e20ee1f8a53bd8decf33c2a74b062edf2f4d9a62eec424","instanceIndex":-1,"ip":"104.154.201.157","latency":"0.412692s","line":[{"logMessage":"IdentityContextLogger logUserAndHisProperties: [Identity][BEGIN] userId:userId userMail:e@mail.com companyId:companyId isSI:false\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-02-20T00:59:59.265Z"},{"logMessage":"StoreReviewJSON init: e@mail.com ( user ) is viewing StoreReviewJSON\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-02-20T00:59:59.277Z"},{"logMessage":"StoreReviewJSON respond: type is BOSS_EVALUATION\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-02-20T00:59:59.278Z"},{"logMessage":"DraftService storeDraft: tim.rehm@radius.com is creating new draft type BOSS_EVALUATION, length=(92/0/0/0/0/0/0/0/0/0/0/)\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-02-20T00:59:59.306Z"},{"logMessage":"ReviewService editManagerEvaluation: new manager's review of matt@company.com foo.com: <div><br></div><div><br></div><div>Overall - [...]\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-02-20T00:59:59.404Z"},{"logMessage":"ReviewService editManagerEvaluation: basic manager ratings are changed\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-02-20T00:59:59.405Z"},{"logMessage":"ReviewService editManagerEvaluation: manager-review item 0 is changed (0->92 chars)\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-02-20T00:59:59.406Z"},{"logMessage":"ReviewService editManagerEvaluation: new boss x:-1 and new boss y:-1\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-02-20T00:59:59.411Z"},{"logMessage":"ActionBroker post: Dispatching ReviewManagerChangedAction\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-02-20T00:59:59.451Z"},{"logMessage":"RemoteAddressService getRemoteAddr: Request is forwarded by proxy '104.154.201.157'. Remote Address of client is '4.16.194.178, 35.186.210.178'\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-02-20T00:59:59.455Z"},{"logMessage":"AuditRecordService trackAuditRecord: tracking auditRecord REVIEW_EDIT_MANAGERASSESSMENT for companyID *jNS-sWGTzKWrvR4xcy46w: edited the manager's-assessment of the review {userName} in {containerName}\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-02-20T00:59:59.457Z"},{"logMessage":"AppEngineQueue enqueueBatch: 1 task(s) of /api/tasks/reviews/stats/cycle enqueued\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-02-20T00:59:59.603Z"},{"logMessage":"IdentityContextLogger logContextEnd: [Identity END]\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-02-20T00:59:59.604Z"}],"megaCycles":"241","method":"POST","requestId":"58aa3f8f00ff03c6042eabdadf0001737e736d616c6c2d696d70726f76656d656e74732d687264000133000100","resource":"/storeReviewJSON?csrfToken=de190133f12807935ed240369fd102a924128001f4eb911344eab432267514fa","responseSize":"1003","startTime":"2017-02-20T00:59:59.2473Z","status":200,"urlMapEntry":"unused","userAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36","versionId":"3"},"resource":{"labels":{"module_id":"default","project_id":"small-improvements-hrd","version_id":"3","zone":"us2"},"type":"gae_app"},"severity":"INFO","timestamp":"2017-02-20T00:59:59.2473Z"}

data/spec/filters/google_appengine_spec.rb

@@ -1,43 +1,60 @@
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/filters/google_appengine"
 require "logstash/json"
-require 'digest'
 
 describe LogStash::Filters::GoogleAppengine do
 
-  md5 = Digest::MD5.new
-
   config <<-CONFIG
         filter {
           google_appengine { }
          }
   CONFIG
 
-  describe "should merge the request payload with the reuest lines data" do
+  describe "should merge the request payload with the request lines data" do
     test_sample = LogStash::Json::load(File.open("spec/filters/appengine.logs.json", "rb").read)
     sample (test_sample) do
       insist { subject.length } == 3
 
       insist { subject[0].get("message") }=="IdentityFilter logUserIdentity: [[meta]] <anonymous:true>\n"
-      insist { subject[0].get("id") }== md5.hexdigest(subject[0].get("requestId") + "0")
+      insist { subject[0].get("id") }== subject[0].get("requestId") + "0"
       insist { subject[0].get("time") } == "2015-09-03T10:59:40.589Z"
       insist { subject[0].get("position") } == 0
       insist { subject[0].get("@type") } == nil
 
       insist { subject[1].get("message") }=="HttpOnlyFilter getSession: add additional Set-Cookie with httpOnly-flag for JSESSIONID\n"
-      insist { subject[1].get("id") } == md5.hexdigest(subject[1].get("requestId") + "1")
+      insist { subject[1].get("id") } == subject[1].get("requestId") + "1"
       insist { subject[1].get("@type") } == nil
       insist { subject[1].get("time") } =="2015-09-03T10:59:40.65Z"
       insist { subject[1].get("position") } == 1
     end
   end
 
+  describe "should readv2 log data" do
+    test_sample = LogStash::Json::load(File.open("spec/filters/appengine.logs-v2.json", "rb").read)
+    sample (test_sample) do
+      insist { subject.length } == 13
+
+      insist { subject[0].get("message") }=="IdentityContextLogger logUserAndHisProperties: [Identity][BEGIN] userId:userId userMail:e@mail.com companyId:companyId isSI:false\n"
+      insist { subject[0].get("id") }== subject[0].get("requestId") + "0"
+      insist { subject[0].get("time") } == "2017-02-20T00:59:59.265Z"
+      insist { subject[0].get("position") } == 0
+      insist { subject[0].get("@type") } == nil
+
+      insist { subject[1].get("message") } == "StoreReviewJSON init: e@mail.com ( user ) is viewing StoreReviewJSON\n"
+      insist { subject[1].get("id") } == subject[1].get("requestId") + "1"
+      insist { subject[1].get("@type") } == nil
+      insist { subject[1].get("time") } =="2017-02-20T00:59:59.277Z"
+      insist { subject[1].get("position") } == 1
+    end
+  end
+
+
   describe "should handle logs even when they have no lines" do
     test_sample = LogStash::Json.load(File.open("spec/filters/appengine.logs-without-lines.json", "rb").read)
     sample (test_sample) do
 
       insist { subject.get("resource") } == "/images/website/welcome/keyFeatures/objectives.jpg"
-      insist { subject.get("id") } == md5.hexdigest(subject.get("requestId"))
+      insist { subject.get("id") } == subject.get("requestId")
       insist { subject.get("time") } == subject.get("endTime")
       insist { subject.get("@type") } == nil
     end

metadata

@@ -1,63 +1,49 @@
 --- !ruby/object:Gem::Specification
 name: logstash-filter-google_appengine
 version: !ruby/object:Gem::Version
-  version: 0.120.0
+  version: 0.121.0
 platform: ruby
 authors:
 - Small Improvements
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2017-01-18 00:00:00.000000000 Z
+date: 2017-02-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: logstash-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 5.0.0
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - '='
+        version: '1.60'
+    - - "<="
       - !ruby/object:Gem::Version
-        version: 5.0.0
+        version: '2.99'
+  name: logstash-core-plugin-api
   prerelease: false
   type: :runtime
-- !ruby/object:Gem::Dependency
-  name: logstash-devutils
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 1.0.0
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: 2.0.0
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.0.0
-    - - "<"
+        version: '1.60'
+    - - "<="
       - !ruby/object:Gem::Version
-        version: 2.0.0
-  prerelease: false
-  type: :development
+        version: '2.99'
 - !ruby/object:Gem::Dependency
-  name: logstash-core-plugin-api
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 2.1.17
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: 2.1.17
-  prerelease: false
-  type: :development
+        version: '0'
 description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
 email: mruhwedel@small-improvements.com
 executables: []
@@ -71,6 +57,7 @@ files:
 - Rakefile
 - lib/logstash/filters/google_appengine.rb
 - logstash-filter-google_appengine.gemspec
+- spec/filters/appengine.logs-v2.json
 - spec/filters/appengine.logs-without-lines.json
 - spec/filters/appengine.logs.json
 - spec/filters/google_appengine_spec.rb
@@ -103,6 +90,7 @@ signing_key:
 specification_version: 4
 summary: This filter may be used to decode via inputs appengine logs
 test_files:
+- spec/filters/appengine.logs-v2.json
 - spec/filters/appengine.logs-without-lines.json
 - spec/filters/appengine.logs.json
 - spec/filters/google_appengine_spec.rb