logstash-filter-google_appengine 0.120.0 → 0.121.0
Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a3b95e78210a10d77a1042ed9d7a598bcd9d577a
|
4
|
+
data.tar.gz: 83eee1233cf84a75374c1f64984c3ef9b8ecfa71
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 3675bf66f7ff2b80afc9c7303a51e777a244bfbeb2b435f0dce665cec426319ce9a40934c8055b2bf76d72e01d2060369aa2342759c764b09e74f67413adf018
|
7
|
+
data.tar.gz: b625103c5261ecaecae991d48c55ac29bde8143f859f492febd6cb63f039303a503bf3ce6b521feb480a8a8b881c522583bae6db8c53c388962ea45a59bcb624
|
@@ -1,16 +1,14 @@
|
|
1
1
|
# encoding: utf-8
|
2
2
|
require "logstash/filters/base"
|
3
3
|
require "logstash/namespace"
|
4
|
-
require 'digest'
|
5
4
|
|
6
5
|
class LogStash::Filters::GoogleAppengine < LogStash::Filters::Base
|
7
6
|
|
8
7
|
config_name "google_appengine"
|
9
8
|
|
10
9
|
public
|
10
|
+
|
11
11
|
def register
|
12
|
-
@md5 = Digest::MD5.new
|
13
|
-
@semaphore = Mutex.new
|
14
12
|
end
|
15
13
|
|
16
14
|
def filter(event)
|
@@ -43,7 +41,7 @@ class LogStash::Filters::GoogleAppengine < LogStash::Filters::Base
|
|
43
41
|
|
44
42
|
def collect_line_data(i, line, payload)
|
45
43
|
{
|
46
|
-
'id' =>
|
44
|
+
'id' => (payload['requestId'] + i.to_s),
|
47
45
|
'message' => line.delete('logMessage'),
|
48
46
|
'position' => i
|
49
47
|
}
|
@@ -54,7 +52,7 @@ class LogStash::Filters::GoogleAppengine < LogStash::Filters::Base
|
|
54
52
|
# noinspection RubyStringKeysInHashInspection
|
55
53
|
def collect_resource_request_data(payload)
|
56
54
|
{
|
57
|
-
'id' =>
|
55
|
+
'id' => payload['requestId'],
|
58
56
|
'time' => payload['endTime'],
|
59
57
|
'position' => 0
|
60
58
|
}
|
@@ -67,10 +65,4 @@ class LogStash::Filters::GoogleAppengine < LogStash::Filters::Base
|
|
67
65
|
new_event
|
68
66
|
end
|
69
67
|
|
70
|
-
def get_id(source)
|
71
|
-
@semaphore.synchronize {
|
72
|
-
@md5.hexdigest(source) #md5 isn't threadsafe :(
|
73
|
-
}
|
74
|
-
end
|
75
|
-
|
76
68
|
end # class LogStash::Filters::GoogleAppengine
|
@@ -1,7 +1,7 @@
|
|
1
1
|
Gem::Specification.new do |s|
|
2
2
|
|
3
3
|
s.name = 'logstash-filter-google_appengine'
|
4
|
-
s.version = '0.
|
4
|
+
s.version = '0.121.0'
|
5
5
|
s.licenses = ['Apache License (2.0)']
|
6
6
|
s.summary = "This filter may be used to decode via inputs appengine logs"
|
7
7
|
s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
|
@@ -16,10 +16,9 @@ Gem::Specification.new do |s|
|
|
16
16
|
|
17
17
|
s.metadata = {"logstash_plugin" => "true", "logstash_group" => "filter"}
|
18
18
|
|
19
|
-
|
20
|
-
|
19
|
+
s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
|
20
|
+
|
21
|
+
s.add_development_dependency 'logstash-devutils'
|
21
22
|
|
22
|
-
s.add_development_dependency 'logstash-devutils', '>= 1.0.0', '< 2.0.0'
|
23
|
-
s.add_development_dependency 'logstash-core-plugin-api', '2.1.17'
|
24
23
|
end
|
25
24
|
|
@@ -0,0 +1 @@
|
|
1
|
+
{"httpRequest":{"status":200},"insertId":"58aa3f900007437ae2d40d60","labels":{"clone_id":"00c61b117cfd455a65e9c255fe7ad6e20ee1f8a53bd8decf33c2a74b062edf2f4d9a62eec424"},"logName":"projects/small-improvements-hrd/logs/appengine.googleapis.com%2Frequest_log","operation":{"first":true,"id":"58aa3f8f00ff03c6042eabdadf0001737e736d616c6c2d696d70726f76656d656e74732d687264000133000100","last":true,"producer":"appengine.googleapis.com/request_id"},"protoPayload":{"@type":"type.googleapis.com/google.appengine.logging.v1.RequestLog","appEngineRelease":"1.9.48","appId":"s~small-improvements-hrd","cost":1.12093e-07,"endTime":"2017-02-20T00:59:59.659992Z","finished":true,"first":true,"host":"radius.small-improvements.com","httpVersion":"HTTP/1.1","instanceId":"00c61b117cfd455a65e9c255fe7ad6e20ee1f8a53bd8decf33c2a74b062edf2f4d9a62eec424","instanceIndex":-1,"ip":"104.154.201.157","latency":"0.412692s","line":[{"logMessage":"IdentityContextLogger logUserAndHisProperties: [Identity][BEGIN] userId:userId userMail:e@mail.com companyId:companyId isSI:false\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-02-20T00:59:59.265Z"},{"logMessage":"StoreReviewJSON init: e@mail.com ( user ) is viewing StoreReviewJSON\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-02-20T00:59:59.277Z"},{"logMessage":"StoreReviewJSON respond: type is BOSS_EVALUATION\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-02-20T00:59:59.278Z"},{"logMessage":"DraftService storeDraft: tim.rehm@radius.com is creating new draft type BOSS_EVALUATION, length=(92/0/0/0/0/0/0/0/0/0/0/)\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-02-20T00:59:59.306Z"},{"logMessage":"ReviewService editManagerEvaluation: new manager's review of matt@company.com foo.com: <div><br></div><div><br></div><div>Overall - [...]\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-02-20T00:59:59.404Z"},{"logMessage":"ReviewService editManagerEvaluation: basic manager ratings are changed\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-02-20T00:59:59.405Z"},{"logMessage":"ReviewService editManagerEvaluation: manager-review item 0 is changed (0->92 chars)\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-02-20T00:59:59.406Z"},{"logMessage":"ReviewService editManagerEvaluation: new boss x:-1 and new boss y:-1\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-02-20T00:59:59.411Z"},{"logMessage":"ActionBroker post: Dispatching ReviewManagerChangedAction\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-02-20T00:59:59.451Z"},{"logMessage":"RemoteAddressService getRemoteAddr: Request is forwarded by proxy '104.154.201.157'. Remote Address of client is '4.16.194.178, 35.186.210.178'\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-02-20T00:59:59.455Z"},{"logMessage":"AuditRecordService trackAuditRecord: tracking auditRecord REVIEW_EDIT_MANAGERASSESSMENT for companyID *jNS-sWGTzKWrvR4xcy46w: edited the manager's-assessment of the review {userName} in {containerName}\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-02-20T00:59:59.457Z"},{"logMessage":"AppEngineQueue enqueueBatch: 1 task(s) of /api/tasks/reviews/stats/cycle enqueued\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-02-20T00:59:59.603Z"},{"logMessage":"IdentityContextLogger logContextEnd: [Identity END]\n","severity":"INFO","sourceLocation":{"file":"JavaUtilLoggingAppender.java","functionName":"com.praisemanager.util.JavaUtilLoggingAppender.append","line":"25"},"time":"2017-02-20T00:59:59.604Z"}],"megaCycles":"241","method":"POST","requestId":"58aa3f8f00ff03c6042eabdadf0001737e736d616c6c2d696d70726f76656d656e74732d687264000133000100","resource":"/storeReviewJSON?csrfToken=de190133f12807935ed240369fd102a924128001f4eb911344eab432267514fa","responseSize":"1003","startTime":"2017-02-20T00:59:59.2473Z","status":200,"urlMapEntry":"unused","userAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36","versionId":"3"},"resource":{"labels":{"module_id":"default","project_id":"small-improvements-hrd","version_id":"3","zone":"us2"},"type":"gae_app"},"severity":"INFO","timestamp":"2017-02-20T00:59:59.2473Z"}
|
@@ -1,43 +1,60 @@
|
|
1
1
|
require "logstash/devutils/rspec/spec_helper"
|
2
2
|
require "logstash/filters/google_appengine"
|
3
3
|
require "logstash/json"
|
4
|
-
require 'digest'
|
5
4
|
|
6
5
|
describe LogStash::Filters::GoogleAppengine do
|
7
6
|
|
8
|
-
md5 = Digest::MD5.new
|
9
|
-
|
10
7
|
config <<-CONFIG
|
11
8
|
filter {
|
12
9
|
google_appengine { }
|
13
10
|
}
|
14
11
|
CONFIG
|
15
12
|
|
16
|
-
describe "should merge the request payload with the
|
13
|
+
describe "should merge the request payload with the request lines data" do
|
17
14
|
test_sample = LogStash::Json::load(File.open("spec/filters/appengine.logs.json", "rb").read)
|
18
15
|
sample (test_sample) do
|
19
16
|
insist { subject.length } == 3
|
20
17
|
|
21
18
|
insist { subject[0].get("message") }=="IdentityFilter logUserIdentity: [[meta]] <anonymous:true>\n"
|
22
|
-
insist { subject[0].get("id") }==
|
19
|
+
insist { subject[0].get("id") }== subject[0].get("requestId") + "0"
|
23
20
|
insist { subject[0].get("time") } == "2015-09-03T10:59:40.589Z"
|
24
21
|
insist { subject[0].get("position") } == 0
|
25
22
|
insist { subject[0].get("@type") } == nil
|
26
23
|
|
27
24
|
insist { subject[1].get("message") }=="HttpOnlyFilter getSession: add additional Set-Cookie with httpOnly-flag for JSESSIONID\n"
|
28
|
-
insist { subject[1].get("id") } ==
|
25
|
+
insist { subject[1].get("id") } == subject[1].get("requestId") + "1"
|
29
26
|
insist { subject[1].get("@type") } == nil
|
30
27
|
insist { subject[1].get("time") } =="2015-09-03T10:59:40.65Z"
|
31
28
|
insist { subject[1].get("position") } == 1
|
32
29
|
end
|
33
30
|
end
|
34
31
|
|
32
|
+
describe "should readv2 log data" do
|
33
|
+
test_sample = LogStash::Json::load(File.open("spec/filters/appengine.logs-v2.json", "rb").read)
|
34
|
+
sample (test_sample) do
|
35
|
+
insist { subject.length } == 13
|
36
|
+
|
37
|
+
insist { subject[0].get("message") }=="IdentityContextLogger logUserAndHisProperties: [Identity][BEGIN] userId:userId userMail:e@mail.com companyId:companyId isSI:false\n"
|
38
|
+
insist { subject[0].get("id") }== subject[0].get("requestId") + "0"
|
39
|
+
insist { subject[0].get("time") } == "2017-02-20T00:59:59.265Z"
|
40
|
+
insist { subject[0].get("position") } == 0
|
41
|
+
insist { subject[0].get("@type") } == nil
|
42
|
+
|
43
|
+
insist { subject[1].get("message") } == "StoreReviewJSON init: e@mail.com ( user ) is viewing StoreReviewJSON\n"
|
44
|
+
insist { subject[1].get("id") } == subject[1].get("requestId") + "1"
|
45
|
+
insist { subject[1].get("@type") } == nil
|
46
|
+
insist { subject[1].get("time") } =="2017-02-20T00:59:59.277Z"
|
47
|
+
insist { subject[1].get("position") } == 1
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
|
35
52
|
describe "should handle logs even when they have no lines" do
|
36
53
|
test_sample = LogStash::Json.load(File.open("spec/filters/appengine.logs-without-lines.json", "rb").read)
|
37
54
|
sample (test_sample) do
|
38
55
|
|
39
56
|
insist { subject.get("resource") } == "/images/website/welcome/keyFeatures/objectives.jpg"
|
40
|
-
insist { subject.get("id") } ==
|
57
|
+
insist { subject.get("id") } == subject.get("requestId")
|
41
58
|
insist { subject.get("time") } == subject.get("endTime")
|
42
59
|
insist { subject.get("@type") } == nil
|
43
60
|
end
|
metadata
CHANGED
@@ -1,63 +1,49 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: logstash-filter-google_appengine
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.121.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Small Improvements
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-
|
11
|
+
date: 2017-02-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
|
-
name: logstash-core
|
15
14
|
requirement: !ruby/object:Gem::Requirement
|
16
15
|
requirements:
|
17
|
-
- -
|
16
|
+
- - ">="
|
18
17
|
- !ruby/object:Gem::Version
|
19
|
-
version:
|
20
|
-
|
21
|
-
requirements:
|
22
|
-
- - '='
|
18
|
+
version: '1.60'
|
19
|
+
- - "<="
|
23
20
|
- !ruby/object:Gem::Version
|
24
|
-
version:
|
21
|
+
version: '2.99'
|
22
|
+
name: logstash-core-plugin-api
|
25
23
|
prerelease: false
|
26
24
|
type: :runtime
|
27
|
-
- !ruby/object:Gem::Dependency
|
28
|
-
name: logstash-devutils
|
29
|
-
requirement: !ruby/object:Gem::Requirement
|
30
|
-
requirements:
|
31
|
-
- - ">="
|
32
|
-
- !ruby/object:Gem::Version
|
33
|
-
version: 1.0.0
|
34
|
-
- - "<"
|
35
|
-
- !ruby/object:Gem::Version
|
36
|
-
version: 2.0.0
|
37
25
|
version_requirements: !ruby/object:Gem::Requirement
|
38
26
|
requirements:
|
39
27
|
- - ">="
|
40
28
|
- !ruby/object:Gem::Version
|
41
|
-
version: 1.
|
42
|
-
- - "
|
29
|
+
version: '1.60'
|
30
|
+
- - "<="
|
43
31
|
- !ruby/object:Gem::Version
|
44
|
-
version: 2.
|
45
|
-
prerelease: false
|
46
|
-
type: :development
|
32
|
+
version: '2.99'
|
47
33
|
- !ruby/object:Gem::Dependency
|
48
|
-
name: logstash-core-plugin-api
|
49
34
|
requirement: !ruby/object:Gem::Requirement
|
50
35
|
requirements:
|
51
|
-
- -
|
36
|
+
- - ">="
|
52
37
|
- !ruby/object:Gem::Version
|
53
|
-
version:
|
38
|
+
version: '0'
|
39
|
+
name: logstash-devutils
|
40
|
+
prerelease: false
|
41
|
+
type: :development
|
54
42
|
version_requirements: !ruby/object:Gem::Requirement
|
55
43
|
requirements:
|
56
|
-
- -
|
44
|
+
- - ">="
|
57
45
|
- !ruby/object:Gem::Version
|
58
|
-
version:
|
59
|
-
prerelease: false
|
60
|
-
type: :development
|
46
|
+
version: '0'
|
61
47
|
description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
|
62
48
|
email: mruhwedel@small-improvements.com
|
63
49
|
executables: []
|
@@ -71,6 +57,7 @@ files:
|
|
71
57
|
- Rakefile
|
72
58
|
- lib/logstash/filters/google_appengine.rb
|
73
59
|
- logstash-filter-google_appengine.gemspec
|
60
|
+
- spec/filters/appengine.logs-v2.json
|
74
61
|
- spec/filters/appengine.logs-without-lines.json
|
75
62
|
- spec/filters/appengine.logs.json
|
76
63
|
- spec/filters/google_appengine_spec.rb
|
@@ -103,6 +90,7 @@ signing_key:
|
|
103
90
|
specification_version: 4
|
104
91
|
summary: This filter may be used to decode via inputs appengine logs
|
105
92
|
test_files:
|
93
|
+
- spec/filters/appengine.logs-v2.json
|
106
94
|
- spec/filters/appengine.logs-without-lines.json
|
107
95
|
- spec/filters/appengine.logs.json
|
108
96
|
- spec/filters/google_appengine_spec.rb
|