logstash-filter-google_appengine 0.119.0 → 0.120.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA1:
3
- metadata.gz: 556aca9dd64a5a7d6aac9cd3174d6f59d50d0fb6
4
- data.tar.gz: 13b97ac6a47a23669f7ff33eb229624347f88ec0
3
+ metadata.gz: d611a1687fbd6223a5fddc1369fd0dd87c85a142
4
+ data.tar.gz: 4d31825089d96b056217accf706425073e019cb7
5
5
  SHA512:
6
- metadata.gz: 0ea38a0548948593c9af677c9a2b86671ad7dcdc3ed377c47967807b9456efe15b3412486e1c60f2ab724c60d8b039eae7fbc94f493975d25029a29d647a6423
7
- data.tar.gz: d5a7a384a5363dd66c3516dc58bec31be25009cff02a229f57a6c50c0918e670e5ac53d1b3890a50e81d192c9578f664acdd9606e965e6163c72950d76055426
6
+ metadata.gz: f6d922c64b3a9e64a299e4f1dba5621ae3252c5fa2674c1d358a682822a70f85c62c182b692324da837771775e1ae16cf25f22bf7c4f9d254ba6d084a3f662cf
7
+ data.tar.gz: fc8620ce2f83a84eda9d5a587e844007c5b3e19b1aadd86453e1d5c1ed9f93e0362a7bf00373d2ca511f1a7a2e649701e686901ca8a422384446861e1692203b
data/.gitignore CHANGED
@@ -84,3 +84,5 @@ com_crashlytics_export_strings.xml
84
84
  crashlytics.properties
85
85
  crashlytics-build.properties
86
86
 
87
+ .mvn
88
+ Gemfile.lock
data/README.md CHANGED
@@ -20,7 +20,12 @@ Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/log
20
20
  ### 1. Plugin Developement and Testing
21
21
 
22
22
  #### Code
23
- - To get started, you'll need JRuby with the Bundler gem installed.
23
+ - To get started, you'll need to install [JRuby](http://jruby.org/). Regular Ruby will not work.
24
+
25
+ - After installing Ruby, install the bundler gem
26
+ ```sh
27
+ gem install bundler
28
+ ```
24
29
 
25
30
  - Create a new plugin or clone and existing from the GitHub [logstash-plugins](https://github.com/logstash-plugins) organization. We also provide [example plugins](https://github.com/logstash-plugins?query=example).
26
31
 
@@ -40,7 +45,7 @@ bundle install
40
45
  - Run tests
41
46
 
42
47
  ```sh
43
- bundle exec rspec
48
+ bundle exec rspec spec
44
49
  ```
45
50
 
46
51
  ### 2. Running your unpublished Plugin in Logstash
@@ -21,9 +21,8 @@ class LogStash::Filters::GoogleAppengine < LogStash::Filters::Base
21
21
  payload.delete '@type'
22
22
  payload['type'] = event.get('type')
23
23
  payload['latencyS'] = to_number(payload['latency'])
24
- if payload['pendingTime']
25
- payload['pendingTimeS'] = to_number(payload['pendingTime'])
26
- end
24
+ payload['pendingTimeS'] = to_number(payload['pendingTime'])
25
+ payload['appTimeS'] = payload['latencyS'] - payload['pendingTimeS']
27
26
  lines = payload.delete 'line'
28
27
 
29
28
  if lines
@@ -1,7 +1,7 @@
1
1
  Gem::Specification.new do |s|
2
2
 
3
3
  s.name = 'logstash-filter-google_appengine'
4
- s.version = '0.119.0'
4
+ s.version = '0.120.0'
5
5
  s.licenses = ['Apache License (2.0)']
6
6
  s.summary = "This filter may be used to decode via inputs appengine logs"
7
7
  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -16,8 +16,10 @@ Gem::Specification.new do |s|
16
16
 
17
17
  s.metadata = {"logstash_plugin" => "true", "logstash_group" => "filter"}
18
18
 
19
- s.add_runtime_dependency "logstash-core", '>= 5.0.0', '< 6.0.0'
19
+ #s.add_runtime_dependency "logstash-core", '>= 5.0.0', '< 6.0.0'
20
+ s.add_runtime_dependency "logstash-core", '5.0.0'
20
21
 
21
- s.add_development_dependency 'logstash-devutils', "~> 0"
22
+ s.add_development_dependency 'logstash-devutils', '>= 1.0.0', '< 2.0.0'
23
+ s.add_development_dependency 'logstash-core-plugin-api', '2.1.17'
22
24
  end
23
25
 
@@ -14,38 +14,66 @@ describe LogStash::Filters::GoogleAppengine do
14
14
  CONFIG
15
15
 
16
16
  describe "should merge the request payload with the reuest lines data" do
17
- test_sample = LogStash::Json::load(File.open("spec/filters/appengine.logs.jsonl", "rb").read)
17
+ test_sample = LogStash::Json::load(File.open("spec/filters/appengine.logs.json", "rb").read)
18
18
  sample (test_sample) do
19
- LATENCY_OF_REQUEST = 0.115752
20
19
  insist { subject.length } == 3
21
20
 
22
- insist { subject[0]["message"] }=="IdentityFilter logUserIdentity: [[meta]] <anonymous:true>\n"
23
- insist { subject[0]["id"] }== md5.hexdigest(subject[0]["requestId"] + "0")
24
- insist { subject[0]["time"] } == "2015-09-03T10:59:40.589Z"
25
- insist { subject[0]["position"] } == 0
26
- insist { subject[0]["@type"] } == nil
27
- insist { subject[0]["latencyS"] } == LATENCY_OF_REQUEST
28
- insist { subject[0]["pendingTimeS"] } == nil
29
-
30
- insist { subject[1]["message"] }=="HttpOnlyFilter getSession: add additional Set-Cookie with httpOnly-flag for JSESSIONID\n"
31
- insist { subject[1]["id"] } == md5.hexdigest(subject[1]["requestId"] + "1")
32
- insist { subject[1]["@type"] } == nil
33
- insist { subject[1]["time"] } =="2015-09-03T10:59:40.65Z"
34
- insist { subject[1]["position"] } == 1
35
- insist { subject[0]["pendingTimeS"] } == nil
21
+ insist { subject[0].get("message") }=="IdentityFilter logUserIdentity: [[meta]] <anonymous:true>\n"
22
+ insist { subject[0].get("id") }== md5.hexdigest(subject[0].get("requestId") + "0")
23
+ insist { subject[0].get("time") } == "2015-09-03T10:59:40.589Z"
24
+ insist { subject[0].get("position") } == 0
25
+ insist { subject[0].get("@type") } == nil
26
+
27
+ insist { subject[1].get("message") }=="HttpOnlyFilter getSession: add additional Set-Cookie with httpOnly-flag for JSESSIONID\n"
28
+ insist { subject[1].get("id") } == md5.hexdigest(subject[1].get("requestId") + "1")
29
+ insist { subject[1].get("@type") } == nil
30
+ insist { subject[1].get("time") } =="2015-09-03T10:59:40.65Z"
31
+ insist { subject[1].get("position") } == 1
36
32
  end
37
33
  end
38
34
 
39
35
  describe "should handle logs even when they have no lines" do
40
- test_sample = LogStash::Json.load(File.open("spec/filters/appengine.logs-without-lines.jsonl", "rb").read)
36
+ test_sample = LogStash::Json.load(File.open("spec/filters/appengine.logs-without-lines.json", "rb").read)
37
+ sample (test_sample) do
38
+
39
+ insist { subject.get("resource") } == "/images/website/welcome/keyFeatures/objectives.jpg"
40
+ insist { subject.get("id") } == md5.hexdigest(subject.get("requestId"))
41
+ insist { subject.get("time") } == subject.get("endTime")
42
+ insist { subject.get("@type") } == nil
43
+ end
44
+ end
45
+
46
+ describe "should convert latency (string) to latencyS (number)" do
47
+ test_sample = LogStash::Json.load(File.open("spec/filters/log-with-pendingTime.json", "rb").read)
48
+ sample (test_sample) do
49
+ insist { subject[0].get("latency") } == "0.779603s"
50
+ insist { subject[0].get("latencyS") } == 0.779603
51
+ end
52
+ end
53
+
54
+ describe "should convert pendingTime (string) to pendingTimeS (number)" do
55
+ test_sample = LogStash::Json.load(File.open("spec/filters/log-with-pendingTime.json", "rb").read)
41
56
  sample (test_sample) do
57
+ insist { subject[0].get("pendingTime") } == "0.712152958s"
58
+ insist { subject[0].get("pendingTimeS") } == 0.712152958
59
+ end
60
+ end
42
61
 
43
- insist { subject["resource"] } == "/images/website/welcome/keyFeatures/objectives.jpg"
44
- insist { subject["id"] } == md5.hexdigest(subject["requestId"])
45
- insist { subject["time"] } == subject["endTime"]
46
- insist { subject["@type"] } == nil
62
+ describe "should convert missing pendingTime to pendingTimeS is 0" do
63
+ test_sample = LogStash::Json.load(File.open("spec/filters/log-without-pendingTime.json", "rb").read)
64
+ sample (test_sample) do
65
+ insist { subject[0].get("pendingTime") } == nil
66
+ insist { subject[0].get("pendingTimeS") } == 0
47
67
  end
48
68
  end
69
+
70
+ describe "should calculate appTimeS from latencyS and pendingTimeS" do
71
+ test_sample = LogStash::Json.load(File.open("spec/filters/log-with-pendingTime.json", "rb").read)
72
+ sample (test_sample) do
73
+ insist { subject[0].get("appTimeS") } == 0.779603 - 0.712152958
74
+ end
75
+ end
76
+
49
77
  end
50
78
 
51
79
 
@@ -0,0 +1 @@
1
+ {"httpRequest":{"status":200},"insertId":"587e51b60003881392cdafdb","log":"appengine.googleapis.com/request_log","metadata":{"labels":{"appengine.googleapis.com/clone_id":"00c61b117c4f6c766c245383b11b010fd60fe8947ea802c068ad8db2eb45c60880b2ed170e6cb624","appengine.googleapis.com/module_id":"default","appengine.googleapis.com/request_id":"587e51b500ff0688e147d97bf70001737e736d616c6c2d696d70726f76656d656e74732d687264000133000100","appengine.googleapis.com/version_id":"3","clone_id":"00c61b117c4f6c766c245383b11b010fd60fe8947ea802c068ad8db2eb45c60880b2ed170e6cb624","module_id":"default","request_id":"587e51b500ff0688e147d97bf70001737e736d616c6c2d696d70726f76656d656e74732d687264000133000100","version_id":"3"},"projectId":"small-improvements-hrd","serviceName":"appengine.googleapis.com","severity":"INFO","timestamp":"2017-01-17T17:17:41.428257Z","zone":"us2"},"operation":{"first":true,"id":"587e51b500ff0688e147d97bf70001737e736d616c6c2d696d70726f76656d656e74732d687264000133000100","last":true,"producer":"appengine.googleapis.com/request_id"},"protoPayload":{"@type":"type.googleapis.com/google.appengine.logging.v1.RequestLog","appEngineRelease":"1.9.48","appId":"s~small-improvements-hrd","cost":6.0908000000000005e-08,"endTime":"2017-01-17T17:17:42.20786Z","finished":true,"first":true,"host":"customer.small-improvements.com","httpVersion":"HTTP/1.1","instanceId":"00c61b117c4f6c766c245383b11b010fd60fe8947ea802c068ad8db2eb45c60880b2ed170e6cb624","instanceIndex":-1,"ip":"65.25.254.214","latency":"0.779603s","line":[{"logMessage":"IdentityContextLogger logUserAndHisProperties: [Identity][BEGIN] userId:*tf83mvBtXCwmMpvkyrvTS userMail:john.doe@customer.com companyId:inpfVlYp1rug1ndpA7r7TV isSI:false\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-01-17T17:17:42.195Z"},{"logMessage":"IdentityContextLogger logContextEnd: [Identity END]\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-01-17T17:17:42.206Z"}],"megaCycles":"39","method":"GET","pendingTime":"0.712152958s","requestId":"587e51b500ff0688e147d97bf70001737e736d616c6c2d696d70726f76656d656e74732d687264000133000100","resource":"/api/avatars/2/DT.svg?size=50","responseSize":"545","startTime":"2017-01-17T17:17:41.428257Z","status":200,"urlMapEntry":"unused","userAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.95 Safari/537.36","versionId":"3"}}
@@ -0,0 +1 @@
1
+ {"httpRequest":{"status":200},"insertId":"587e4d90000cdd3a0ec11b34","log":"appengine.googleapis.com/request_log","metadata":{"labels":{"appengine.googleapis.com/clone_id":"00c61b117cf26d77f54f5c83731f597b2b82d3856b7706be25b95a1657b7fc4903828a568855b3f7","appengine.googleapis.com/module_id":"default","appengine.googleapis.com/request_id":"587e4d9000ff03f6c7ed2936820001737e736d616c6c2d696d70726f76656d656e74732d687264000133000100","appengine.googleapis.com/version_id":"3","clone_id":"00c61b117cf26d77f54f5c83731f597b2b82d3856b7706be25b95a1657b7fc4903828a568855b3f7","module_id":"default","request_id":"587e4d9000ff03f6c7ed2936820001737e736d616c6c2d696d70726f76656d656e74732d687264000133000100","version_id":"3"},"projectId":"small-improvements-hrd","serviceName":"appengine.googleapis.com","severity":"INFO","timestamp":"2017-01-17T17:00:00.259783Z","zone":"us2"},"operation":{"first":true,"id":"587e4d9000ff03f6c7ed2936820001737e736d616c6c2d696d70726f76656d656e74732d687264000133000100","last":true,"producer":"appengine.googleapis.com/request_id"},"protoPayload":{"@type":"type.googleapis.com/google.appengine.logging.v1.RequestLog","appEngineRelease":"1.9.48","appId":"s~small-improvements-hrd","cost":1.3321599999999999e-07,"endTime":"2017-01-17T17:00:00.421235Z","finished":true,"first":true,"host":"customer.small-improvements.com","httpVersion":"HTTP/1.1","instanceId":"00c61b117cf26d77f54f5c83731f597b2b82d3856b7706be25b95a1657b7fc4903828a568855b3f7","instanceIndex":-1,"ip":"209.152.64.69","latency":"0.161452s","line":[{"logMessage":"IdentityContextLogger logUserAndHisProperties: [Identity][BEGIN] userId:7LudpFWKJAP9Ze3kixSICQ userMail:alice@customer.com companyId:ZjxohdxgW3w6yfLXibPTq6 isSI:false\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-01-17T17:00:00.289Z"},{"logMessage":"MetricsFlushingFilter acquireLockAndSendMetrics: Flush interval of PT10S exceeded sending metrics now\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-01-17T17:00:00.299Z"},{"logMessage":"IdentityContextLogger logContextEnd: [Identity END]\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-01-17T17:00:00.419Z"}],"megaCycles":"57","method":"GET","referrer":"https://customer.small-improvements.com/","requestId":"587e4d9000ff03f6c7ed2936820001737e736d616c6c2d696d70726f76656d656e74732d687264000133000100","resource":"/api/avatars/23/AR.svg","responseSize":"550","startTime":"2017-01-17T17:00:00.259783Z","status":200,"traceId":"cde94892157fbf26b5144f6db20b34a8","urlMapEntry":"unused","userAgent":"Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; TNJB; rv:11.0) like Gecko","versionId":"3"}}
metadata CHANGED
@@ -1,47 +1,61 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: logstash-filter-google_appengine
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.119.0
4
+ version: 0.120.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Small Improvements
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2016-10-28 00:00:00.000000000 Z
11
+ date: 2017-01-18 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: logstash-core
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - '='
18
+ - !ruby/object:Gem::Version
19
+ version: 5.0.0
15
20
  version_requirements: !ruby/object:Gem::Requirement
16
21
  requirements:
17
- - - ">="
22
+ - - '='
18
23
  - !ruby/object:Gem::Version
19
24
  version: 5.0.0
25
+ prerelease: false
26
+ type: :runtime
27
+ - !ruby/object:Gem::Dependency
28
+ name: logstash-devutils
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - ">="
32
+ - !ruby/object:Gem::Version
33
+ version: 1.0.0
20
34
  - - "<"
21
35
  - !ruby/object:Gem::Version
22
- version: 6.0.0
23
- requirement: !ruby/object:Gem::Requirement
36
+ version: 2.0.0
37
+ version_requirements: !ruby/object:Gem::Requirement
24
38
  requirements:
25
39
  - - ">="
26
40
  - !ruby/object:Gem::Version
27
- version: 5.0.0
41
+ version: 1.0.0
28
42
  - - "<"
29
43
  - !ruby/object:Gem::Version
30
- version: 6.0.0
44
+ version: 2.0.0
31
45
  prerelease: false
32
- type: :runtime
46
+ type: :development
33
47
  - !ruby/object:Gem::Dependency
34
- name: logstash-devutils
35
- version_requirements: !ruby/object:Gem::Requirement
48
+ name: logstash-core-plugin-api
49
+ requirement: !ruby/object:Gem::Requirement
36
50
  requirements:
37
- - - "~>"
51
+ - - '='
38
52
  - !ruby/object:Gem::Version
39
- version: '0'
40
- requirement: !ruby/object:Gem::Requirement
53
+ version: 2.1.17
54
+ version_requirements: !ruby/object:Gem::Requirement
41
55
  requirements:
42
- - - "~>"
56
+ - - '='
43
57
  - !ruby/object:Gem::Version
44
- version: '0'
58
+ version: 2.1.17
45
59
  prerelease: false
46
60
  type: :development
47
61
  description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
@@ -57,9 +71,11 @@ files:
57
71
  - Rakefile
58
72
  - lib/logstash/filters/google_appengine.rb
59
73
  - logstash-filter-google_appengine.gemspec
60
- - spec/filters/appengine.logs-without-lines.jsonl
61
- - spec/filters/appengine.logs.jsonl
74
+ - spec/filters/appengine.logs-without-lines.json
75
+ - spec/filters/appengine.logs.json
62
76
  - spec/filters/google_appengine_spec.rb
77
+ - spec/filters/log-with-pendingTime.json
78
+ - spec/filters/log-without-pendingTime.json
63
79
  homepage: https://github.com/SmallImprovements/logstash-filter-google_appengine
64
80
  licenses:
65
81
  - Apache License (2.0)
@@ -82,11 +98,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
82
98
  version: '0'
83
99
  requirements: []
84
100
  rubyforge_project:
85
- rubygems_version: 2.4.8
101
+ rubygems_version: 2.6.8
86
102
  signing_key:
87
103
  specification_version: 4
88
104
  summary: This filter may be used to decode via inputs appengine logs
89
105
  test_files:
90
- - spec/filters/appengine.logs-without-lines.jsonl
91
- - spec/filters/appengine.logs.jsonl
106
+ - spec/filters/appengine.logs-without-lines.json
107
+ - spec/filters/appengine.logs.json
92
108
  - spec/filters/google_appengine_spec.rb
109
+ - spec/filters/log-with-pendingTime.json
110
+ - spec/filters/log-without-pendingTime.json