logstash-filter-google_appengine 0.119.0 → 0.120.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +2 -0
- data/README.md +7 -2
- data/lib/logstash/filters/google_appengine.rb +2 -3
- data/logstash-filter-google_appengine.gemspec +5 -3
- data/spec/filters/{appengine.logs-without-lines.jsonl → appengine.logs-without-lines.json} +0 -0
- data/spec/filters/{appengine.logs.jsonl → appengine.logs.json} +0 -0
- data/spec/filters/google_appengine_spec.rb +49 -21
- data/spec/filters/log-with-pendingTime.json +1 -0
- data/spec/filters/log-without-pendingTime.json +1 -0
- metadata +38 -20
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d611a1687fbd6223a5fddc1369fd0dd87c85a142
|
4
|
+
data.tar.gz: 4d31825089d96b056217accf706425073e019cb7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: f6d922c64b3a9e64a299e4f1dba5621ae3252c5fa2674c1d358a682822a70f85c62c182b692324da837771775e1ae16cf25f22bf7c4f9d254ba6d084a3f662cf
|
7
|
+
data.tar.gz: fc8620ce2f83a84eda9d5a587e844007c5b3e19b1aadd86453e1d5c1ed9f93e0362a7bf00373d2ca511f1a7a2e649701e686901ca8a422384446861e1692203b
|
data/.gitignore
CHANGED
data/README.md
CHANGED
@@ -20,7 +20,12 @@ Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/log
|
|
20
20
|
### 1. Plugin Developement and Testing
|
21
21
|
|
22
22
|
#### Code
|
23
|
-
- To get started, you'll need JRuby
|
23
|
+
- To get started, you'll need to install [JRuby](http://jruby.org/). Regular Ruby will not work.
|
24
|
+
|
25
|
+
- After installing Ruby, install the bundler gem
|
26
|
+
```sh
|
27
|
+
gem install bundler
|
28
|
+
```
|
24
29
|
|
25
30
|
- Create a new plugin or clone and existing from the GitHub [logstash-plugins](https://github.com/logstash-plugins) organization. We also provide [example plugins](https://github.com/logstash-plugins?query=example).
|
26
31
|
|
@@ -40,7 +45,7 @@ bundle install
|
|
40
45
|
- Run tests
|
41
46
|
|
42
47
|
```sh
|
43
|
-
bundle exec rspec
|
48
|
+
bundle exec rspec spec
|
44
49
|
```
|
45
50
|
|
46
51
|
### 2. Running your unpublished Plugin in Logstash
|
@@ -21,9 +21,8 @@ class LogStash::Filters::GoogleAppengine < LogStash::Filters::Base
|
|
21
21
|
payload.delete '@type'
|
22
22
|
payload['type'] = event.get('type')
|
23
23
|
payload['latencyS'] = to_number(payload['latency'])
|
24
|
-
|
25
|
-
|
26
|
-
end
|
24
|
+
payload['pendingTimeS'] = to_number(payload['pendingTime'])
|
25
|
+
payload['appTimeS'] = payload['latencyS'] - payload['pendingTimeS']
|
27
26
|
lines = payload.delete 'line'
|
28
27
|
|
29
28
|
if lines
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-filter-google_appengine'
|
4
|
-
s.version = '0.
|
4
|
+
s.version = '0.120.0'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "This filter may be used to decode via inputs appengine logs"
|
7
7
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
@@ -16,8 +16,10 @@ Gem::Specification.new do |s|
|
|
16
16
|
|
17
17
|
s.metadata = {"logstash_plugin" => "true", "logstash_group" => "filter"}
|
18
18
|
|
19
|
-
s.add_runtime_dependency "logstash-core", '>= 5.0.0', '< 6.0.0'
|
19
|
+
#s.add_runtime_dependency "logstash-core", '>= 5.0.0', '< 6.0.0'
|
20
|
+
s.add_runtime_dependency "logstash-core", '5.0.0'
|
20
21
|
|
21
|
-
s.add_development_dependency 'logstash-devutils',
|
22
|
+
s.add_development_dependency 'logstash-devutils', '>= 1.0.0', '< 2.0.0'
|
23
|
+
s.add_development_dependency 'logstash-core-plugin-api', '2.1.17'
|
22
24
|
end
|
23
25
|
|
File without changes
|
File without changes
|
@@ -14,38 +14,66 @@ describe LogStash::Filters::GoogleAppengine do
|
|
14
14
|
CONFIG
|
15
15
|
|
16
16
|
describe "should merge the request payload with the reuest lines data" do
|
17
|
-
test_sample = LogStash::Json::load(File.open("spec/filters/appengine.logs.
|
17
|
+
test_sample = LogStash::Json::load(File.open("spec/filters/appengine.logs.json", "rb").read)
|
18
18
|
sample (test_sample) do
|
19
|
-
LATENCY_OF_REQUEST = 0.115752
|
20
19
|
insist { subject.length } == 3
|
21
20
|
|
22
|
-
insist { subject[0]
|
23
|
-
insist { subject[0]
|
24
|
-
insist { subject[0]
|
25
|
-
insist { subject[0]
|
26
|
-
insist { subject[0]
|
27
|
-
|
28
|
-
insist { subject[
|
29
|
-
|
30
|
-
insist { subject[1]
|
31
|
-
insist { subject[1]
|
32
|
-
insist { subject[1]
|
33
|
-
insist { subject[1]["time"] } =="2015-09-03T10:59:40.65Z"
|
34
|
-
insist { subject[1]["position"] } == 1
|
35
|
-
insist { subject[0]["pendingTimeS"] } == nil
|
21
|
+
insist { subject[0].get("message") }=="IdentityFilter logUserIdentity: [[meta]] <anonymous:true>\n"
|
22
|
+
insist { subject[0].get("id") }== md5.hexdigest(subject[0].get("requestId") + "0")
|
23
|
+
insist { subject[0].get("time") } == "2015-09-03T10:59:40.589Z"
|
24
|
+
insist { subject[0].get("position") } == 0
|
25
|
+
insist { subject[0].get("@type") } == nil
|
26
|
+
|
27
|
+
insist { subject[1].get("message") }=="HttpOnlyFilter getSession: add additional Set-Cookie with httpOnly-flag for JSESSIONID\n"
|
28
|
+
insist { subject[1].get("id") } == md5.hexdigest(subject[1].get("requestId") + "1")
|
29
|
+
insist { subject[1].get("@type") } == nil
|
30
|
+
insist { subject[1].get("time") } =="2015-09-03T10:59:40.65Z"
|
31
|
+
insist { subject[1].get("position") } == 1
|
36
32
|
end
|
37
33
|
end
|
38
34
|
|
39
35
|
describe "should handle logs even when they have no lines" do
|
40
|
-
test_sample = LogStash::Json.load(File.open("spec/filters/appengine.logs-without-lines.
|
36
|
+
test_sample = LogStash::Json.load(File.open("spec/filters/appengine.logs-without-lines.json", "rb").read)
|
37
|
+
sample (test_sample) do
|
38
|
+
|
39
|
+
insist { subject.get("resource") } == "/images/website/welcome/keyFeatures/objectives.jpg"
|
40
|
+
insist { subject.get("id") } == md5.hexdigest(subject.get("requestId"))
|
41
|
+
insist { subject.get("time") } == subject.get("endTime")
|
42
|
+
insist { subject.get("@type") } == nil
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
describe "should convert latency (string) to latencyS (number)" do
|
47
|
+
test_sample = LogStash::Json.load(File.open("spec/filters/log-with-pendingTime.json", "rb").read)
|
48
|
+
sample (test_sample) do
|
49
|
+
insist { subject[0].get("latency") } == "0.779603s"
|
50
|
+
insist { subject[0].get("latencyS") } == 0.779603
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
describe "should convert pendingTime (string) to pendingTimeS (number)" do
|
55
|
+
test_sample = LogStash::Json.load(File.open("spec/filters/log-with-pendingTime.json", "rb").read)
|
41
56
|
sample (test_sample) do
|
57
|
+
insist { subject[0].get("pendingTime") } == "0.712152958s"
|
58
|
+
insist { subject[0].get("pendingTimeS") } == 0.712152958
|
59
|
+
end
|
60
|
+
end
|
42
61
|
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
insist { subject["
|
62
|
+
describe "should convert missing pendingTime to pendingTimeS is 0" do
|
63
|
+
test_sample = LogStash::Json.load(File.open("spec/filters/log-without-pendingTime.json", "rb").read)
|
64
|
+
sample (test_sample) do
|
65
|
+
insist { subject[0].get("pendingTime") } == nil
|
66
|
+
insist { subject[0].get("pendingTimeS") } == 0
|
47
67
|
end
|
48
68
|
end
|
69
|
+
|
70
|
+
describe "should calculate appTimeS from latencyS and pendingTimeS" do
|
71
|
+
test_sample = LogStash::Json.load(File.open("spec/filters/log-with-pendingTime.json", "rb").read)
|
72
|
+
sample (test_sample) do
|
73
|
+
insist { subject[0].get("appTimeS") } == 0.779603 - 0.712152958
|
74
|
+
end
|
75
|
+
end
|
76
|
+
|
49
77
|
end
|
50
78
|
|
51
79
|
|
@@ -0,0 +1 @@
|
|
1
|
+
{"httpRequest":{"status":200},"insertId":"587e51b60003881392cdafdb","log":"appengine.googleapis.com/request_log","metadata":{"labels":{"appengine.googleapis.com/clone_id":"00c61b117c4f6c766c245383b11b010fd60fe8947ea802c068ad8db2eb45c60880b2ed170e6cb624","appengine.googleapis.com/module_id":"default","appengine.googleapis.com/request_id":"587e51b500ff0688e147d97bf70001737e736d616c6c2d696d70726f76656d656e74732d687264000133000100","appengine.googleapis.com/version_id":"3","clone_id":"00c61b117c4f6c766c245383b11b010fd60fe8947ea802c068ad8db2eb45c60880b2ed170e6cb624","module_id":"default","request_id":"587e51b500ff0688e147d97bf70001737e736d616c6c2d696d70726f76656d656e74732d687264000133000100","version_id":"3"},"projectId":"small-improvements-hrd","serviceName":"appengine.googleapis.com","severity":"INFO","timestamp":"2017-01-17T17:17:41.428257Z","zone":"us2"},"operation":{"first":true,"id":"587e51b500ff0688e147d97bf70001737e736d616c6c2d696d70726f76656d656e74732d687264000133000100","last":true,"producer":"appengine.googleapis.com/request_id"},"protoPayload":{"@type":"type.googleapis.com/google.appengine.logging.v1.RequestLog","appEngineRelease":"1.9.48","appId":"s~small-improvements-hrd","cost":6.0908000000000005e-08,"endTime":"2017-01-17T17:17:42.20786Z","finished":true,"first":true,"host":"customer.small-improvements.com","httpVersion":"HTTP/1.1","instanceId":"00c61b117c4f6c766c245383b11b010fd60fe8947ea802c068ad8db2eb45c60880b2ed170e6cb624","instanceIndex":-1,"ip":"65.25.254.214","latency":"0.779603s","line":[{"logMessage":"IdentityContextLogger logUserAndHisProperties: [Identity][BEGIN] userId:*tf83mvBtXCwmMpvkyrvTS userMail:john.doe@customer.com companyId:inpfVlYp1rug1ndpA7r7TV isSI:false\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-01-17T17:17:42.195Z"},{"logMessage":"IdentityContextLogger logContextEnd: [Identity END]\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-01-17T17:17:42.206Z"}],"megaCycles":"39","method":"GET","pendingTime":"0.712152958s","requestId":"587e51b500ff0688e147d97bf70001737e736d616c6c2d696d70726f76656d656e74732d687264000133000100","resource":"/api/avatars/2/DT.svg?size=50","responseSize":"545","startTime":"2017-01-17T17:17:41.428257Z","status":200,"urlMapEntry":"unused","userAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.95 Safari/537.36","versionId":"3"}}
|
@@ -0,0 +1 @@
|
|
1
|
+
{"httpRequest":{"status":200},"insertId":"587e4d90000cdd3a0ec11b34","log":"appengine.googleapis.com/request_log","metadata":{"labels":{"appengine.googleapis.com/clone_id":"00c61b117cf26d77f54f5c83731f597b2b82d3856b7706be25b95a1657b7fc4903828a568855b3f7","appengine.googleapis.com/module_id":"default","appengine.googleapis.com/request_id":"587e4d9000ff03f6c7ed2936820001737e736d616c6c2d696d70726f76656d656e74732d687264000133000100","appengine.googleapis.com/version_id":"3","clone_id":"00c61b117cf26d77f54f5c83731f597b2b82d3856b7706be25b95a1657b7fc4903828a568855b3f7","module_id":"default","request_id":"587e4d9000ff03f6c7ed2936820001737e736d616c6c2d696d70726f76656d656e74732d687264000133000100","version_id":"3"},"projectId":"small-improvements-hrd","serviceName":"appengine.googleapis.com","severity":"INFO","timestamp":"2017-01-17T17:00:00.259783Z","zone":"us2"},"operation":{"first":true,"id":"587e4d9000ff03f6c7ed2936820001737e736d616c6c2d696d70726f76656d656e74732d687264000133000100","last":true,"producer":"appengine.googleapis.com/request_id"},"protoPayload":{"@type":"type.googleapis.com/google.appengine.logging.v1.RequestLog","appEngineRelease":"1.9.48","appId":"s~small-improvements-hrd","cost":1.3321599999999999e-07,"endTime":"2017-01-17T17:00:00.421235Z","finished":true,"first":true,"host":"customer.small-improvements.com","httpVersion":"HTTP/1.1","instanceId":"00c61b117cf26d77f54f5c83731f597b2b82d3856b7706be25b95a1657b7fc4903828a568855b3f7","instanceIndex":-1,"ip":"209.152.64.69","latency":"0.161452s","line":[{"logMessage":"IdentityContextLogger logUserAndHisProperties: [Identity][BEGIN] userId:7LudpFWKJAP9Ze3kixSICQ userMail:alice@customer.com companyId:ZjxohdxgW3w6yfLXibPTq6 isSI:false\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-01-17T17:00:00.289Z"},{"logMessage":"MetricsFlushingFilter acquireLockAndSendMetrics: Flush interval of PT10S exceeded sending metrics now\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-01-17T17:00:00.299Z"},{"logMessage":"IdentityContextLogger logContextEnd: [Identity END]\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-01-17T17:00:00.419Z"}],"megaCycles":"57","method":"GET","referrer":"https://customer.small-improvements.com/","requestId":"587e4d9000ff03f6c7ed2936820001737e736d616c6c2d696d70726f76656d656e74732d687264000133000100","resource":"/api/avatars/23/AR.svg","responseSize":"550","startTime":"2017-01-17T17:00:00.259783Z","status":200,"traceId":"cde94892157fbf26b5144f6db20b34a8","urlMapEntry":"unused","userAgent":"Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; TNJB; rv:11.0) like Gecko","versionId":"3"}}
|
metadata
CHANGED
@@ -1,47 +1,61 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-filter-google_appengine
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.120.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Small Improvements
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2017-01-18 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: logstash-core
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - '='
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: 5.0.0
|
15
20
|
version_requirements: !ruby/object:Gem::Requirement
|
16
21
|
requirements:
|
17
|
-
- -
|
22
|
+
- - '='
|
18
23
|
- !ruby/object:Gem::Version
|
19
24
|
version: 5.0.0
|
25
|
+
prerelease: false
|
26
|
+
type: :runtime
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: logstash-devutils
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - ">="
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: 1.0.0
|
20
34
|
- - "<"
|
21
35
|
- !ruby/object:Gem::Version
|
22
|
-
version:
|
23
|
-
|
36
|
+
version: 2.0.0
|
37
|
+
version_requirements: !ruby/object:Gem::Requirement
|
24
38
|
requirements:
|
25
39
|
- - ">="
|
26
40
|
- !ruby/object:Gem::Version
|
27
|
-
version:
|
41
|
+
version: 1.0.0
|
28
42
|
- - "<"
|
29
43
|
- !ruby/object:Gem::Version
|
30
|
-
version:
|
44
|
+
version: 2.0.0
|
31
45
|
prerelease: false
|
32
|
-
type: :
|
46
|
+
type: :development
|
33
47
|
- !ruby/object:Gem::Dependency
|
34
|
-
name: logstash-
|
35
|
-
|
48
|
+
name: logstash-core-plugin-api
|
49
|
+
requirement: !ruby/object:Gem::Requirement
|
36
50
|
requirements:
|
37
|
-
- -
|
51
|
+
- - '='
|
38
52
|
- !ruby/object:Gem::Version
|
39
|
-
version:
|
40
|
-
|
53
|
+
version: 2.1.17
|
54
|
+
version_requirements: !ruby/object:Gem::Requirement
|
41
55
|
requirements:
|
42
|
-
- -
|
56
|
+
- - '='
|
43
57
|
- !ruby/object:Gem::Version
|
44
|
-
version:
|
58
|
+
version: 2.1.17
|
45
59
|
prerelease: false
|
46
60
|
type: :development
|
47
61
|
description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
|
@@ -57,9 +71,11 @@ files:
|
|
57
71
|
- Rakefile
|
58
72
|
- lib/logstash/filters/google_appengine.rb
|
59
73
|
- logstash-filter-google_appengine.gemspec
|
60
|
-
- spec/filters/appengine.logs-without-lines.
|
61
|
-
- spec/filters/appengine.logs.
|
74
|
+
- spec/filters/appengine.logs-without-lines.json
|
75
|
+
- spec/filters/appengine.logs.json
|
62
76
|
- spec/filters/google_appengine_spec.rb
|
77
|
+
- spec/filters/log-with-pendingTime.json
|
78
|
+
- spec/filters/log-without-pendingTime.json
|
63
79
|
homepage: https://github.com/SmallImprovements/logstash-filter-google_appengine
|
64
80
|
licenses:
|
65
81
|
- Apache License (2.0)
|
@@ -82,11 +98,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
82
98
|
version: '0'
|
83
99
|
requirements: []
|
84
100
|
rubyforge_project:
|
85
|
-
rubygems_version: 2.
|
101
|
+
rubygems_version: 2.6.8
|
86
102
|
signing_key:
|
87
103
|
specification_version: 4
|
88
104
|
summary: This filter may be used to decode via inputs appengine logs
|
89
105
|
test_files:
|
90
|
-
- spec/filters/appengine.logs-without-lines.
|
91
|
-
- spec/filters/appengine.logs.
|
106
|
+
- spec/filters/appengine.logs-without-lines.json
|
107
|
+
- spec/filters/appengine.logs.json
|
92
108
|
- spec/filters/google_appengine_spec.rb
|
109
|
+
- spec/filters/log-with-pendingTime.json
|
110
|
+
- spec/filters/log-without-pendingTime.json
|