resurfaceio-logger 1.9.2 → 2.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/resurfaceio/all.rb +1 -1
- data/lib/resurfaceio/base_logger.rb +51 -17
- data/lib/resurfaceio/http_logger.rb +20 -89
- data/lib/resurfaceio/http_logger_for_rack.rb +5 -2
- data/lib/resurfaceio/http_logger_for_rails.rb +5 -2
- data/lib/resurfaceio/http_message.rb +33 -12
- data/lib/resurfaceio/http_request_impl.rb +1 -1
- data/lib/resurfaceio/http_response_impl.rb +1 -1
- data/lib/resurfaceio/http_rule.rb +1 -1
- data/lib/resurfaceio/http_rules.rb +189 -33
- data/lib/resurfaceio/timer.rb +14 -0
- data/lib/resurfaceio/usage_loggers.rb +4 -4
- metadata +17 -17
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: e64df211f671ce6788896fe8810fbd7942700b42bf730b47b69c2d7f75add11b
|
|
4
|
+
data.tar.gz: 77f39e9cb57d693ce05ac3280e955137605de0fa13e3b4ada5041c3ff9dd21e0
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: d345193859f6cba73a63757a3686b756837e697a034cdf416193c5de2b23cceabaf401af466ee4f98eaa1f709df608c7e6eb817d132e9141bf347ca6230eb61e
|
|
7
|
+
data.tar.gz: 88ffcade0c109fbf0a70e560b68824847f8467abd6fa9c4cef15a59eb0c83ffbcc78142e9c845b08a98cce40ac2826d2a566a03f6cd6e7fa8458ade8d83cbdbd
|
data/lib/resurfaceio/all.rb
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
1
|
# coding: utf-8
|
|
2
|
-
# © 2016-
|
|
2
|
+
# © 2016-2020 Resurface Labs Inc.
|
|
3
3
|
|
|
4
4
|
require 'uri'
|
|
5
5
|
require 'net/http'
|
|
6
6
|
require 'net/https'
|
|
7
|
+
require 'socket'
|
|
7
8
|
require 'zlib'
|
|
8
9
|
require 'resurfaceio/usage_loggers'
|
|
9
10
|
|
|
@@ -11,6 +12,7 @@ class BaseLogger
|
|
|
11
12
|
|
|
12
13
|
def initialize(agent, options = {})
|
|
13
14
|
@agent = agent
|
|
15
|
+
@host = BaseLogger.host_lookup
|
|
14
16
|
@skip_compression = false
|
|
15
17
|
@skip_submission = false
|
|
16
18
|
@version = BaseLogger.version_lookup
|
|
@@ -42,14 +44,21 @@ class BaseLogger
|
|
|
42
44
|
# validate url when present
|
|
43
45
|
unless @url.nil?
|
|
44
46
|
begin
|
|
45
|
-
|
|
47
|
+
@url_parsed = URI.parse(@url)
|
|
48
|
+
raise Exception unless @url_parsed.scheme.include?('http')
|
|
46
49
|
rescue Exception
|
|
47
50
|
@url = nil
|
|
51
|
+
@url_parsed = nil
|
|
48
52
|
@enabled = false
|
|
49
53
|
end
|
|
50
54
|
end
|
|
51
55
|
|
|
56
|
+
# finalize internal properties
|
|
52
57
|
@enableable = !@queue.nil? || !@url.nil?
|
|
58
|
+
@submit_failures = 0
|
|
59
|
+
@submit_failures_lock = Mutex.new
|
|
60
|
+
@submit_successes = 0
|
|
61
|
+
@submit_successes_lock = Mutex.new
|
|
53
62
|
end
|
|
54
63
|
|
|
55
64
|
def agent
|
|
@@ -74,6 +83,10 @@ class BaseLogger
|
|
|
74
83
|
@enabled && UsageLoggers.enabled?
|
|
75
84
|
end
|
|
76
85
|
|
|
86
|
+
def host
|
|
87
|
+
@host
|
|
88
|
+
end
|
|
89
|
+
|
|
77
90
|
def queue
|
|
78
91
|
@queue
|
|
79
92
|
end
|
|
@@ -94,33 +107,44 @@ class BaseLogger
|
|
|
94
107
|
@skip_submission = value
|
|
95
108
|
end
|
|
96
109
|
|
|
97
|
-
def submit(
|
|
98
|
-
if
|
|
99
|
-
|
|
110
|
+
def submit(msg)
|
|
111
|
+
if msg.nil? || @skip_submission || !enabled?
|
|
112
|
+
# do nothing
|
|
100
113
|
elsif @queue
|
|
101
|
-
@queue <<
|
|
102
|
-
|
|
114
|
+
@queue << msg
|
|
115
|
+
@submit_successes_lock.synchronize { @submit_successes += 1 }
|
|
103
116
|
else
|
|
104
117
|
begin
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
@url_connection.use_ssl = @url.include?('https')
|
|
118
|
+
url_connection = Net::HTTP.new(@url_parsed.host, @url_parsed.port)
|
|
119
|
+
url_connection.use_ssl = @url.include?('https')
|
|
108
120
|
request = Net::HTTP::Post.new(@url_parsed.path)
|
|
121
|
+
request.add_field('Content-Type', 'application/json; charset=UTF-8')
|
|
109
122
|
if @skip_compression
|
|
110
|
-
request.body =
|
|
123
|
+
request.body = msg
|
|
111
124
|
else
|
|
112
125
|
request.add_field('Content-Encoding', 'deflated')
|
|
113
|
-
request.body = Zlib::Deflate.deflate(
|
|
126
|
+
request.body = Zlib::Deflate.deflate(msg)
|
|
127
|
+
end
|
|
128
|
+
response = url_connection.request(request)
|
|
129
|
+
if response.code.to_i == 204
|
|
130
|
+
@submit_successes_lock.synchronize { @submit_successes += 1 }
|
|
131
|
+
else
|
|
132
|
+
@submit_failures_lock.synchronize { @submit_failures += 1 }
|
|
114
133
|
end
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
rescue SocketError
|
|
118
|
-
@url_connection = nil
|
|
119
|
-
false
|
|
134
|
+
rescue Exception
|
|
135
|
+
@submit_failures_lock.synchronize { @submit_failures += 1 }
|
|
120
136
|
end
|
|
121
137
|
end
|
|
122
138
|
end
|
|
123
139
|
|
|
140
|
+
def submit_failures
|
|
141
|
+
@submit_failures
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
def submit_successes
|
|
145
|
+
@submit_successes
|
|
146
|
+
end
|
|
147
|
+
|
|
124
148
|
def url
|
|
125
149
|
@url
|
|
126
150
|
end
|
|
@@ -129,6 +153,16 @@ class BaseLogger
|
|
|
129
153
|
@version
|
|
130
154
|
end
|
|
131
155
|
|
|
156
|
+
def self.host_lookup
|
|
157
|
+
dyno = ENV['DYNO']
|
|
158
|
+
return dyno unless dyno.nil?
|
|
159
|
+
begin
|
|
160
|
+
Socket.gethostname
|
|
161
|
+
rescue
|
|
162
|
+
'unknown'
|
|
163
|
+
end
|
|
164
|
+
end
|
|
165
|
+
|
|
132
166
|
def self.version_lookup
|
|
133
167
|
Gem.loaded_specs['resurfaceio-logger'].version.to_s
|
|
134
168
|
end
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
# coding: utf-8
|
|
2
|
-
# © 2016-
|
|
2
|
+
# © 2016-2020 Resurface Labs Inc.
|
|
3
3
|
|
|
4
4
|
require 'json'
|
|
5
5
|
require 'resurfaceio/base_logger'
|
|
@@ -11,53 +11,20 @@ class HttpLogger < BaseLogger
|
|
|
11
11
|
|
|
12
12
|
AGENT = 'http_logger.rb'.freeze
|
|
13
13
|
|
|
14
|
-
@@default_rules = HttpRules.strict_rules
|
|
15
|
-
|
|
16
|
-
def self.default_rules
|
|
17
|
-
@@default_rules
|
|
18
|
-
end
|
|
19
|
-
|
|
20
|
-
def self.default_rules=(val)
|
|
21
|
-
@@default_rules = val.gsub(/^\s*include default\s*$/, '')
|
|
22
|
-
end
|
|
23
|
-
|
|
24
|
-
def self.string_content_type?(s)
|
|
25
|
-
!s.nil? && !(s =~ /^(text\/(html|plain|xml))|(application\/(json|soap|xml|x-www-form-urlencoded))/i).nil?
|
|
26
|
-
end
|
|
27
|
-
|
|
28
14
|
def initialize(options = {})
|
|
29
15
|
super(AGENT, options)
|
|
30
16
|
|
|
31
|
-
#
|
|
17
|
+
# parse specified rules
|
|
32
18
|
if options.respond_to?(:has_key?) && options.has_key?(:rules)
|
|
33
|
-
@rules = options[:rules]
|
|
34
|
-
@rules = @@default_rules unless @rules.strip.length > 0
|
|
19
|
+
@rules = HttpRules.new(options[:rules])
|
|
35
20
|
else
|
|
36
|
-
@rules =
|
|
21
|
+
@rules = HttpRules.new(nil)
|
|
37
22
|
end
|
|
38
23
|
|
|
39
|
-
#
|
|
40
|
-
|
|
41
|
-
@
|
|
42
|
-
@
|
|
43
|
-
@rules_remove = prs.select {|r| 'remove' == r.verb}
|
|
44
|
-
@rules_remove_if = prs.select {|r| 'remove_if' == r.verb}
|
|
45
|
-
@rules_remove_if_found = prs.select {|r| 'remove_if_found' == r.verb}
|
|
46
|
-
@rules_remove_unless = prs.select {|r| 'remove_unless' == r.verb}
|
|
47
|
-
@rules_remove_unless_found = prs.select {|r| 'remove_unless_found' == r.verb}
|
|
48
|
-
@rules_replace = prs.select {|r| 'replace' == r.verb}
|
|
49
|
-
@rules_sample = prs.select {|r| 'sample' == r.verb}
|
|
50
|
-
@rules_stop = prs.select {|r| 'stop' == r.verb}
|
|
51
|
-
@rules_stop_if = prs.select {|r| 'stop_if' == r.verb}
|
|
52
|
-
@rules_stop_if_found = prs.select {|r| 'stop_if_found' == r.verb}
|
|
53
|
-
@rules_stop_unless = prs.select {|r| 'stop_unless' == r.verb}
|
|
54
|
-
@rules_stop_unless_found = prs.select {|r| 'stop_unless_found' == r.verb}
|
|
55
|
-
@skip_compression = prs.select {|r| 'skip_compression' == r.verb}.length > 0
|
|
56
|
-
@skip_submission = prs.select {|r| 'skip_submission' == r.verb}.length > 0
|
|
57
|
-
|
|
58
|
-
# finish validating rules
|
|
59
|
-
raise RuntimeError.new('Multiple sample rules') if @rules_sample.length > 1
|
|
60
|
-
unless @url.nil? || @url.start_with?('https') || @rules_allow_http_url
|
|
24
|
+
# apply configuration rules
|
|
25
|
+
@skip_compression = @rules.skip_compression
|
|
26
|
+
@skip_submission = @rules.skip_submission
|
|
27
|
+
unless @url.nil? || @url.start_with?('https') || @rules.allow_http_url
|
|
61
28
|
@enableable = false
|
|
62
29
|
@enabled = false
|
|
63
30
|
end
|
|
@@ -67,57 +34,21 @@ class HttpLogger < BaseLogger
|
|
|
67
34
|
@rules
|
|
68
35
|
end
|
|
69
36
|
|
|
70
|
-
def
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
def format(request, response, response_body = nil, request_body = nil, now = nil)
|
|
75
|
-
details = HttpMessage.build(request, response, response_body, request_body)
|
|
37
|
+
def submit_if_passing(details)
|
|
38
|
+
# apply active rules
|
|
39
|
+
details = @rules.apply(details)
|
|
40
|
+
return nil if details.nil?
|
|
76
41
|
|
|
77
|
-
#
|
|
78
|
-
unless @rules_copy_session_field.empty?
|
|
79
|
-
ssn = request.session
|
|
80
|
-
if !ssn.nil? && ssn.respond_to?(:keys)
|
|
81
|
-
@rules_copy_session_field.each do |r|
|
|
82
|
-
ssn.keys.each {|d| (details << ["session_field:#{d}", ssn[d].to_s]) if r.param1.match(d)}
|
|
83
|
-
end
|
|
84
|
-
end
|
|
85
|
-
end
|
|
86
|
-
|
|
87
|
-
# quit early based on stop rules if configured
|
|
88
|
-
@rules_stop.each {|r| details.each {|d| return nil if r.scope.match(d[0])}}
|
|
89
|
-
@rules_stop_if_found.each {|r| details.each {|d| return nil if r.scope.match(d[0]) && r.param1.match(d[1])}}
|
|
90
|
-
@rules_stop_if.each {|r| details.each {|d| return nil if r.scope.match(d[0]) && r.param1.match(d[1])}}
|
|
91
|
-
passed = 0
|
|
92
|
-
@rules_stop_unless_found.each {|r| details.each {|d| passed += 1 if r.scope.match(d[0]) && r.param1.match(d[1])}}
|
|
93
|
-
return nil if passed != @rules_stop_unless_found.length
|
|
94
|
-
passed = 0
|
|
95
|
-
@rules_stop_unless.each {|r| details.each {|d| passed += 1 if r.scope.match(d[0]) && r.param1.match(d[1])}}
|
|
96
|
-
return nil if passed != @rules_stop_unless.length
|
|
97
|
-
|
|
98
|
-
# do sampling if configured
|
|
99
|
-
return nil if !@rules_sample[0].nil? && (rand * 100 >= @rules_sample[0].param1)
|
|
100
|
-
|
|
101
|
-
# winnow sensitive details based on remove rules if configured
|
|
102
|
-
@rules_remove.each {|r| details.delete_if {|d| r.scope.match(d[0])}}
|
|
103
|
-
@rules_remove_unless_found.each {|r| details.delete_if {|d| r.scope.match(d[0]) && !r.param1.match(d[1])}}
|
|
104
|
-
@rules_remove_if_found.each {|r| details.delete_if {|d| r.scope.match(d[0]) && r.param1.match(d[1])}}
|
|
105
|
-
@rules_remove_unless.each {|r| details.delete_if {|d| r.scope.match(d[0]) && !r.param1.match(d[1])}}
|
|
106
|
-
@rules_remove_if.each {|r| details.delete_if {|d| r.scope.match(d[0]) && r.param1.match(d[1])}}
|
|
107
|
-
return nil if details.empty?
|
|
108
|
-
|
|
109
|
-
# mask sensitive details based on replace rules if configured
|
|
110
|
-
@rules_replace.each {|r| details.each {|d| d[1] = d[1].gsub(r.param1, r.param2) if r.scope.match(d[0])}}
|
|
111
|
-
|
|
112
|
-
# remove any details with empty values
|
|
113
|
-
details.delete_if {|d| '' == d[1]}
|
|
114
|
-
return nil if details.empty?
|
|
115
|
-
|
|
116
|
-
# finish message
|
|
117
|
-
details << ['now', now.nil? ? (Time.now.to_f * 1000).floor.to_s : now]
|
|
42
|
+
# finalize message
|
|
118
43
|
details << ['agent', @agent]
|
|
44
|
+
details << ['host', @host]
|
|
119
45
|
details << ['version', @version]
|
|
120
|
-
|
|
46
|
+
|
|
47
|
+
# let's do this thing
|
|
48
|
+
submit(JSON.generate(details))
|
|
121
49
|
end
|
|
122
50
|
|
|
51
|
+
def self.string_content_type?(s)
|
|
52
|
+
!s.nil? && !(s =~ /^(text\/(html|plain|xml))|(application\/(json|soap|xml|x-www-form-urlencoded))/i).nil?
|
|
53
|
+
end
|
|
123
54
|
end
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
# coding: utf-8
|
|
2
|
-
# © 2016-
|
|
2
|
+
# © 2016-2020 Resurface Labs Inc.
|
|
3
3
|
|
|
4
4
|
require 'rack'
|
|
5
5
|
require 'resurfaceio/http_logger'
|
|
6
|
+
require 'resurfaceio/http_message'
|
|
7
|
+
require 'resurfaceio/timer'
|
|
6
8
|
|
|
7
9
|
class HttpLoggerForRack # http://rack.rubyforge.org/doc/SPEC.html
|
|
8
10
|
|
|
@@ -16,12 +18,13 @@ class HttpLoggerForRack # http://rack.rubyforge.org/doc/SPEC.html
|
|
|
16
18
|
end
|
|
17
19
|
|
|
18
20
|
def call(env)
|
|
21
|
+
timer = Timer.new
|
|
19
22
|
status, headers, body = @app.call(env)
|
|
20
23
|
if @logger.enabled? && (status < 300 || status == 302)
|
|
21
24
|
response = Rack::Response.new(body, status, headers)
|
|
22
25
|
if HttpLogger::string_content_type?(response.content_type)
|
|
23
26
|
request = Rack::Request.new(env)
|
|
24
|
-
|
|
27
|
+
HttpMessage.send(logger, request, response, nil, nil, nil, timer.millis)
|
|
25
28
|
end
|
|
26
29
|
end
|
|
27
30
|
[status, headers, body]
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
# coding: utf-8
|
|
2
|
-
# © 2016-
|
|
2
|
+
# © 2016-2020 Resurface Labs Inc.
|
|
3
3
|
|
|
4
4
|
require 'resurfaceio/http_logger'
|
|
5
|
+
require 'resurfaceio/http_message'
|
|
6
|
+
require 'resurfaceio/timer'
|
|
5
7
|
|
|
6
8
|
class HttpLoggerForRails
|
|
7
9
|
|
|
@@ -14,13 +16,14 @@ class HttpLoggerForRails
|
|
|
14
16
|
end
|
|
15
17
|
|
|
16
18
|
def around(controller)
|
|
19
|
+
timer = Timer.new
|
|
17
20
|
yield
|
|
18
21
|
if @logger.enabled?
|
|
19
22
|
request = controller.request
|
|
20
23
|
response = controller.response
|
|
21
24
|
status = response.status
|
|
22
25
|
if (status < 300 || status == 302) && HttpLogger::string_content_type?(response.content_type)
|
|
23
|
-
|
|
26
|
+
HttpMessage.send(logger, request, response, nil, nil, nil, timer.millis)
|
|
24
27
|
end
|
|
25
28
|
end
|
|
26
29
|
end
|
|
@@ -1,15 +1,38 @@
|
|
|
1
1
|
# coding: utf-8
|
|
2
|
-
# © 2016-
|
|
2
|
+
# © 2016-2020 Resurface Labs Inc.
|
|
3
3
|
|
|
4
4
|
require 'json'
|
|
5
5
|
|
|
6
6
|
class HttpMessage
|
|
7
7
|
|
|
8
|
+
def self.send(logger, request, response, response_body = nil, request_body = nil, now = nil, interval = nil)
|
|
9
|
+
return unless logger.enabled?
|
|
10
|
+
|
|
11
|
+
# copy details from request & response
|
|
12
|
+
message = build(request, response, response_body, request_body)
|
|
13
|
+
|
|
14
|
+
# copy details from active session
|
|
15
|
+
unless logger.rules.copy_session_field.empty?
|
|
16
|
+
ssn = request.session
|
|
17
|
+
if !ssn.nil? && ssn.respond_to?(:keys)
|
|
18
|
+
logger.rules.copy_session_field.each do |r|
|
|
19
|
+
ssn.keys.each {|d| (message << ["session_field:#{d}", ssn[d].to_s]) if r.param1.match(d)}
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# add timing details
|
|
25
|
+
message << ['now', now.nil? ? (Time.now.to_f * 1000).floor.to_s : now]
|
|
26
|
+
message << ['interval', interval] unless interval.nil?
|
|
27
|
+
|
|
28
|
+
logger.submit_if_passing(message)
|
|
29
|
+
end
|
|
30
|
+
|
|
8
31
|
def self.build(request, response, response_body = nil, request_body = nil)
|
|
9
32
|
message = []
|
|
10
|
-
append_value message, 'request_method', request.request_method
|
|
11
|
-
append_value message, 'request_url', request.url
|
|
12
|
-
append_value message, 'response_code', response.status
|
|
33
|
+
append_value message, 'request_method', request.request_method unless request.request_method.nil?
|
|
34
|
+
append_value message, 'request_url', request.url unless request.url.nil?
|
|
35
|
+
append_value message, 'response_code', response.status unless response.status.nil?
|
|
13
36
|
append_request_headers message, request
|
|
14
37
|
append_request_params message, request
|
|
15
38
|
append_response_headers message, response
|
|
@@ -19,8 +42,6 @@ class HttpMessage
|
|
|
19
42
|
return message
|
|
20
43
|
end
|
|
21
44
|
|
|
22
|
-
protected
|
|
23
|
-
|
|
24
45
|
def self.append_request_headers(message, request)
|
|
25
46
|
respond_to_env = request.respond_to?(:env)
|
|
26
47
|
if respond_to_env || request.respond_to?(:headers)
|
|
@@ -74,12 +95,12 @@ class HttpMessage
|
|
|
74
95
|
unless key.nil?
|
|
75
96
|
unless value.nil?
|
|
76
97
|
case value
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
98
|
+
when Array
|
|
99
|
+
message << [key, value.join]
|
|
100
|
+
when String
|
|
101
|
+
message << [key, value]
|
|
102
|
+
else
|
|
103
|
+
message << [key, value.to_s]
|
|
83
104
|
end
|
|
84
105
|
end
|
|
85
106
|
end
|
|
@@ -1,37 +1,39 @@
|
|
|
1
1
|
# coding: utf-8
|
|
2
|
-
# © 2016-
|
|
2
|
+
# © 2016-2020 Resurface Labs Inc.
|
|
3
3
|
|
|
4
4
|
class HttpRules
|
|
5
5
|
|
|
6
|
+
DEBUG_RULES = "allow_http_url\ncopy_session_field /.*/\n".freeze
|
|
7
|
+
|
|
8
|
+
STANDARD_RULES = %q(/request_header:cookie|response_header:set-cookie/ remove
|
|
9
|
+
/(request|response)_body|request_param/ replace /[a-zA-Z0-9.!#$%&’*+\/=?^_`{|}~-]+@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)/, /x@y.com/
|
|
10
|
+
/request_body|request_param|response_body/ replace /[0-9\.\-\/]{9,}/, /xyxy/
|
|
11
|
+
).freeze
|
|
12
|
+
|
|
13
|
+
STRICT_RULES = %q(/request_url/ replace /([^\?;]+).*/, !\\\\1!
|
|
14
|
+
/request_body|response_body|request_param:.*|request_header:(?!user-agent).*|response_header:(?!(content-length)|(content-type)).*/ remove
|
|
15
|
+
).freeze
|
|
16
|
+
|
|
17
|
+
@@default_rules = STRICT_RULES
|
|
18
|
+
|
|
19
|
+
def self.default_rules
|
|
20
|
+
@@default_rules
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def self.default_rules=(val)
|
|
24
|
+
@@default_rules = val.gsub(/^\s*include default\s*$/, '')
|
|
25
|
+
end
|
|
26
|
+
|
|
6
27
|
def self.debug_rules
|
|
7
|
-
|
|
28
|
+
DEBUG_RULES
|
|
8
29
|
end
|
|
9
30
|
|
|
10
31
|
def self.standard_rules
|
|
11
|
-
|
|
12
|
-
/(request|response)_body|request_param/ replace /[a-zA-Z0-9.!#$%&’*+\/=?^_`{|}~-]+@[a-zA-Z0-9-]+(?:\.[a-zA-Z0-9-]+)/, /x@y.com/
|
|
13
|
-
/request_body|request_param|response_body/ replace /[0-9\.\-\/]{9,}/, /xyxy/
|
|
14
|
-
)
|
|
32
|
+
STANDARD_RULES
|
|
15
33
|
end
|
|
16
34
|
|
|
17
35
|
def self.strict_rules
|
|
18
|
-
|
|
19
|
-
/request_body|response_body|request_param:.*|request_header:(?!user-agent).*|response_header:(?!(content-length)|(content-type)).*/ remove
|
|
20
|
-
)
|
|
21
|
-
end
|
|
22
|
-
|
|
23
|
-
def self.parse(rules)
|
|
24
|
-
result = []
|
|
25
|
-
unless rules.nil?
|
|
26
|
-
rules = rules.gsub(/^\s*include debug\s*$/, debug_rules)
|
|
27
|
-
rules = rules.gsub(/^\s*include standard\s*$/, standard_rules)
|
|
28
|
-
rules = rules.gsub(/^\s*include strict\s*$/, strict_rules)
|
|
29
|
-
rules.each_line do |rule|
|
|
30
|
-
parsed = parse_rule(rule)
|
|
31
|
-
result << parsed unless parsed.nil?
|
|
32
|
-
end
|
|
33
|
-
end
|
|
34
|
-
result
|
|
36
|
+
STRICT_RULES
|
|
35
37
|
end
|
|
36
38
|
|
|
37
39
|
def self.parse_rule(r)
|
|
@@ -76,15 +78,13 @@ class HttpRules
|
|
|
76
78
|
end
|
|
77
79
|
end
|
|
78
80
|
|
|
79
|
-
protected
|
|
80
|
-
|
|
81
81
|
def self.parse_regex(r, regex)
|
|
82
|
-
|
|
83
|
-
raise RuntimeError.new("Invalid regex (#{regex}) in rule: #{r}") if '*' ==
|
|
84
|
-
|
|
85
|
-
|
|
82
|
+
s = parse_string(r, regex)
|
|
83
|
+
raise RuntimeError.new("Invalid regex (#{regex}) in rule: #{r}") if '*' == s || '+' == s || '?' == s
|
|
84
|
+
s = "^#{s}" unless s.start_with?('^')
|
|
85
|
+
s = "#{s}$" unless s.end_with?('$')
|
|
86
86
|
begin
|
|
87
|
-
return Regexp.compile(
|
|
87
|
+
return Regexp.compile(s)
|
|
88
88
|
rescue RegexpError
|
|
89
89
|
raise RuntimeError.new("Invalid regex (#{regex}) in rule: #{r}")
|
|
90
90
|
end
|
|
@@ -98,15 +98,171 @@ class HttpRules
|
|
|
98
98
|
end
|
|
99
99
|
end
|
|
100
100
|
|
|
101
|
-
def self.parse_string(r,
|
|
101
|
+
def self.parse_string(r, expr)
|
|
102
102
|
%w(~ ! % | /).each do |sep|
|
|
103
|
-
if (m =
|
|
103
|
+
if (m = expr.match(/^[#{sep}](.*)[#{sep}]$/))
|
|
104
104
|
m1 = m[1]
|
|
105
105
|
raise RuntimeError.new("Unescaped separator (#{sep}) in rule: #{r}") if m1.match(/^[#{sep}].*|.*[^\\][#{sep}].*/)
|
|
106
106
|
return m1.gsub("\\#{sep}", sep)
|
|
107
107
|
end
|
|
108
108
|
end
|
|
109
|
-
raise RuntimeError.new("Invalid expression (#{
|
|
109
|
+
raise RuntimeError.new("Invalid expression (#{expr}) in rule: #{r}")
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
def initialize(rules)
|
|
113
|
+
rules = HttpRules.default_rules if rules.nil?
|
|
114
|
+
|
|
115
|
+
# load rules from external files
|
|
116
|
+
if rules.start_with?('file://')
|
|
117
|
+
rfile = rules[7..]
|
|
118
|
+
begin
|
|
119
|
+
rules = File.read(rfile)
|
|
120
|
+
rescue
|
|
121
|
+
raise RuntimeError.new("Failed to load rules: #{rfile}")
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
# force default rules if necessary
|
|
126
|
+
rules = rules.gsub(/^\s*include default\s*$/, HttpRules.default_rules)
|
|
127
|
+
rules = HttpRules.default_rules unless rules.strip.length > 0
|
|
128
|
+
|
|
129
|
+
# expand rule inclues
|
|
130
|
+
rules = rules.gsub(/^\s*include debug\s*$/, DEBUG_RULES)
|
|
131
|
+
rules = rules.gsub(/^\s*include standard\s*$/, STANDARD_RULES)
|
|
132
|
+
rules = rules.gsub(/^\s*include strict\s*$/, STRICT_RULES)
|
|
133
|
+
@text = rules
|
|
134
|
+
|
|
135
|
+
# parse all rules
|
|
136
|
+
prs = []
|
|
137
|
+
rules.each_line do |rule|
|
|
138
|
+
parsed = HttpRules.parse_rule(rule)
|
|
139
|
+
prs << parsed unless parsed.nil?
|
|
140
|
+
end
|
|
141
|
+
@length = prs.length
|
|
142
|
+
|
|
143
|
+
# break out rules by verb
|
|
144
|
+
@allow_http_url = prs.select {|r| 'allow_http_url' == r.verb}.length > 0
|
|
145
|
+
@copy_session_field = prs.select {|r| 'copy_session_field' == r.verb}
|
|
146
|
+
@remove = prs.select {|r| 'remove' == r.verb}
|
|
147
|
+
@remove_if = prs.select {|r| 'remove_if' == r.verb}
|
|
148
|
+
@remove_if_found = prs.select {|r| 'remove_if_found' == r.verb}
|
|
149
|
+
@remove_unless = prs.select {|r| 'remove_unless' == r.verb}
|
|
150
|
+
@remove_unless_found = prs.select {|r| 'remove_unless_found' == r.verb}
|
|
151
|
+
@replace = prs.select {|r| 'replace' == r.verb}
|
|
152
|
+
@sample = prs.select {|r| 'sample' == r.verb}
|
|
153
|
+
@skip_compression = prs.select {|r| 'skip_compression' == r.verb}.length > 0
|
|
154
|
+
@skip_submission = prs.select {|r| 'skip_submission' == r.verb}.length > 0
|
|
155
|
+
@stop = prs.select {|r| 'stop' == r.verb}
|
|
156
|
+
@stop_if = prs.select {|r| 'stop_if' == r.verb}
|
|
157
|
+
@stop_if_found = prs.select {|r| 'stop_if_found' == r.verb}
|
|
158
|
+
@stop_unless = prs.select {|r| 'stop_unless' == r.verb}
|
|
159
|
+
@stop_unless_found = prs.select {|r| 'stop_unless_found' == r.verb}
|
|
160
|
+
|
|
161
|
+
# validate rules
|
|
162
|
+
raise RuntimeError.new('Multiple sample rules') if @sample.length > 1
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
def allow_http_url
|
|
166
|
+
@allow_http_url
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
def copy_session_field
|
|
170
|
+
@copy_session_field
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
def length
|
|
174
|
+
@length
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
def remove
|
|
178
|
+
@remove
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
def remove_if
|
|
182
|
+
@remove_if
|
|
183
|
+
end
|
|
184
|
+
|
|
185
|
+
def remove_if_found
|
|
186
|
+
@remove_if_found
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
def remove_unless
|
|
190
|
+
@remove_unless
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
def remove_unless_found
|
|
194
|
+
@remove_unless_found
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
def replace
|
|
198
|
+
@replace
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
def sample
|
|
202
|
+
@sample
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
def skip_compression
|
|
206
|
+
@skip_compression
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
def skip_submission
|
|
210
|
+
@skip_submission
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
def stop
|
|
214
|
+
@stop
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
def stop_if
|
|
218
|
+
@stop_if
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
def stop_if_found
|
|
222
|
+
@stop_if_found
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
def stop_unless
|
|
226
|
+
@stop_unless
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
def stop_unless_found
|
|
230
|
+
@stop_unless_found
|
|
231
|
+
end
|
|
232
|
+
|
|
233
|
+
def text
|
|
234
|
+
@text
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
def apply(details)
|
|
238
|
+
# stop rules come first
|
|
239
|
+
@stop.each {|r| details.each {|d| return nil if r.scope.match(d[0])}}
|
|
240
|
+
@stop_if_found.each {|r| details.each {|d| return nil if r.scope.match(d[0]) && r.param1.match(d[1])}}
|
|
241
|
+
@stop_if.each {|r| details.each {|d| return nil if r.scope.match(d[0]) && r.param1.match(d[1])}}
|
|
242
|
+
passed = 0
|
|
243
|
+
@stop_unless_found.each {|r| details.each {|d| passed += 1 if r.scope.match(d[0]) && r.param1.match(d[1])}}
|
|
244
|
+
return nil if passed != @stop_unless_found.length
|
|
245
|
+
passed = 0
|
|
246
|
+
@stop_unless.each {|r| details.each {|d| passed += 1 if r.scope.match(d[0]) && r.param1.match(d[1])}}
|
|
247
|
+
return nil if passed != @stop_unless.length
|
|
248
|
+
|
|
249
|
+
# do sampling if configured
|
|
250
|
+
return nil if !@sample[0].nil? && (rand * 100 >= @sample[0].param1)
|
|
251
|
+
|
|
252
|
+
# winnow sensitive details based on remove rules if configured
|
|
253
|
+
@remove.each {|r| details.delete_if {|d| r.scope.match(d[0])}}
|
|
254
|
+
@remove_unless_found.each {|r| details.delete_if {|d| r.scope.match(d[0]) && !r.param1.match(d[1])}}
|
|
255
|
+
@remove_if_found.each {|r| details.delete_if {|d| r.scope.match(d[0]) && r.param1.match(d[1])}}
|
|
256
|
+
@remove_unless.each {|r| details.delete_if {|d| r.scope.match(d[0]) && !r.param1.match(d[1])}}
|
|
257
|
+
@remove_if.each {|r| details.delete_if {|d| r.scope.match(d[0]) && r.param1.match(d[1])}}
|
|
258
|
+
return nil if details.empty?
|
|
259
|
+
|
|
260
|
+
# mask sensitive details based on replace rules if configured
|
|
261
|
+
@replace.each {|r| details.each {|d| d[1] = d[1].gsub(r.param1, r.param2) if r.scope.match(d[0])}}
|
|
262
|
+
|
|
263
|
+
# remove any details with empty values
|
|
264
|
+
details.delete_if {|d| '' == d[1]}
|
|
265
|
+
details.empty? ? nil : details
|
|
110
266
|
end
|
|
111
267
|
|
|
112
268
|
REGEX_ALLOW_HTTP_URL = /^\s*allow_http_url\s*(#.*)?$/.freeze
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
# coding: utf-8
|
|
2
|
+
# © 2016-2020 Resurface Labs Inc.
|
|
3
|
+
|
|
4
|
+
class Timer
|
|
5
|
+
|
|
6
|
+
def initialize
|
|
7
|
+
@started = Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_millisecond)
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def millis
|
|
11
|
+
(Process.clock_gettime(Process::CLOCK_MONOTONIC, :float_millisecond) - @started).to_s
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
end
|
|
@@ -1,18 +1,18 @@
|
|
|
1
1
|
# coding: utf-8
|
|
2
|
-
# © 2016-
|
|
2
|
+
# © 2016-2020 Resurface Labs Inc.
|
|
3
3
|
|
|
4
4
|
class UsageLoggers
|
|
5
5
|
|
|
6
|
-
@@
|
|
6
|
+
@@BRICKED = 'true'.eql?(ENV['USAGE_LOGGERS_DISABLE'])
|
|
7
7
|
|
|
8
|
-
@@disabled = @@
|
|
8
|
+
@@disabled = @@BRICKED
|
|
9
9
|
|
|
10
10
|
def self.disable
|
|
11
11
|
@@disabled = true
|
|
12
12
|
end
|
|
13
13
|
|
|
14
14
|
def self.enable
|
|
15
|
-
@@disabled = false unless @@
|
|
15
|
+
@@disabled = false unless @@BRICKED
|
|
16
16
|
end
|
|
17
17
|
|
|
18
18
|
def self.enabled?
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: resurfaceio-logger
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version:
|
|
4
|
+
version: 2.0.3
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- RobDickinson
|
|
8
|
-
autorequire:
|
|
8
|
+
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2020-07-30 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: bundler
|
|
@@ -16,58 +16,58 @@ dependencies:
|
|
|
16
16
|
requirements:
|
|
17
17
|
- - "~>"
|
|
18
18
|
- !ruby/object:Gem::Version
|
|
19
|
-
version: '1.
|
|
19
|
+
version: '1.17'
|
|
20
20
|
type: :development
|
|
21
21
|
prerelease: false
|
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
|
23
23
|
requirements:
|
|
24
24
|
- - "~>"
|
|
25
25
|
- !ruby/object:Gem::Version
|
|
26
|
-
version: '1.
|
|
26
|
+
version: '1.17'
|
|
27
27
|
- !ruby/object:Gem::Dependency
|
|
28
28
|
name: rack
|
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
|
30
30
|
requirements:
|
|
31
31
|
- - "~>"
|
|
32
32
|
- !ruby/object:Gem::Version
|
|
33
|
-
version: '
|
|
33
|
+
version: '2.2'
|
|
34
34
|
type: :development
|
|
35
35
|
prerelease: false
|
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
|
37
37
|
requirements:
|
|
38
38
|
- - "~>"
|
|
39
39
|
- !ruby/object:Gem::Version
|
|
40
|
-
version: '
|
|
40
|
+
version: '2.2'
|
|
41
41
|
- !ruby/object:Gem::Dependency
|
|
42
42
|
name: rake
|
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
|
44
44
|
requirements:
|
|
45
45
|
- - "~>"
|
|
46
46
|
- !ruby/object:Gem::Version
|
|
47
|
-
version: '
|
|
47
|
+
version: '13.0'
|
|
48
48
|
type: :development
|
|
49
49
|
prerelease: false
|
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
|
51
51
|
requirements:
|
|
52
52
|
- - "~>"
|
|
53
53
|
- !ruby/object:Gem::Version
|
|
54
|
-
version: '
|
|
54
|
+
version: '13.0'
|
|
55
55
|
- !ruby/object:Gem::Dependency
|
|
56
56
|
name: rspec
|
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
|
58
58
|
requirements:
|
|
59
59
|
- - "~>"
|
|
60
60
|
- !ruby/object:Gem::Version
|
|
61
|
-
version: '3.
|
|
61
|
+
version: '3.9'
|
|
62
62
|
type: :development
|
|
63
63
|
prerelease: false
|
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
|
65
65
|
requirements:
|
|
66
66
|
- - "~>"
|
|
67
67
|
- !ruby/object:Gem::Version
|
|
68
|
-
version: '3.
|
|
68
|
+
version: '3.9'
|
|
69
69
|
description: Library for usage logging
|
|
70
|
-
email:
|
|
70
|
+
email:
|
|
71
71
|
executables: []
|
|
72
72
|
extensions: []
|
|
73
73
|
extra_rdoc_files: []
|
|
@@ -82,12 +82,13 @@ files:
|
|
|
82
82
|
- lib/resurfaceio/http_response_impl.rb
|
|
83
83
|
- lib/resurfaceio/http_rule.rb
|
|
84
84
|
- lib/resurfaceio/http_rules.rb
|
|
85
|
+
- lib/resurfaceio/timer.rb
|
|
85
86
|
- lib/resurfaceio/usage_loggers.rb
|
|
86
87
|
homepage: https://github.com/resurfaceio/logger-ruby
|
|
87
88
|
licenses:
|
|
88
89
|
- Apache-2.0
|
|
89
90
|
metadata: {}
|
|
90
|
-
post_install_message:
|
|
91
|
+
post_install_message:
|
|
91
92
|
rdoc_options: []
|
|
92
93
|
require_paths:
|
|
93
94
|
- lib
|
|
@@ -95,16 +96,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
|
95
96
|
requirements:
|
|
96
97
|
- - "~>"
|
|
97
98
|
- !ruby/object:Gem::Version
|
|
98
|
-
version: '2.
|
|
99
|
+
version: '2.6'
|
|
99
100
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
100
101
|
requirements:
|
|
101
102
|
- - ">="
|
|
102
103
|
- !ruby/object:Gem::Version
|
|
103
104
|
version: '0'
|
|
104
105
|
requirements: []
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
signing_key:
|
|
106
|
+
rubygems_version: 3.0.3
|
|
107
|
+
signing_key:
|
|
108
108
|
specification_version: 4
|
|
109
109
|
summary: Library for usage logging
|
|
110
110
|
test_files: []
|