active_record-sql_analyzer 0.0.8 → 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +1 -0
- data/lib/active_record/sql_analyzer/configuration.rb +6 -6
- data/lib/active_record/sql_analyzer/monkeypatches/query.rb +40 -2
- data/lib/active_record/sql_analyzer/version.rb +1 -1
- data/spec/active_record/sql_analyzer/end_to_end_spec.rb +98 -2
- data/spec/active_record/sql_analyzer/redacted_logger_spec.rb +49 -12
- metadata +3 -4
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 48238f31887e49755dbac5b74348f9f580dc0b65
|
4
|
+
data.tar.gz: 7c875b3aa9dfdb7e4449f12893ff1ddf2f7effff
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ceaaefc3ea471c547e81c4ce227a7fb1b1b13e6d61ea5caffd6d1391b1802fe35af2f5948d7624f1c376e75bfc0c4a89b8be6efb61667ddb5e1faac0bc34e33b
|
7
|
+
data.tar.gz: a89a8a5ae0d6e38c08123a64b0e1a2ec63b8fcfd8641e5eaff55507c122717177c97fc84ab50f605e08ee3a0954ef98a711abc8fa6188253b76ae2ff359cd05c
|
data/Gemfile
CHANGED
@@ -146,19 +146,19 @@ module ActiveRecord
|
|
146
146
|
end
|
147
147
|
|
148
148
|
def setup_defaults
|
149
|
-
quotedValuePattern =
|
149
|
+
quotedValuePattern = %{('([^\\\\']|\\\\.|'')*'|"([^\\\\"]|\\\\.|"")*")}
|
150
150
|
@options[:sql_redactors] = [
|
151
151
|
Redactor.new(/\n/, " "),
|
152
152
|
Redactor.new(/\s+/, " "),
|
153
|
-
Redactor.new(/
|
153
|
+
Redactor.new(/IN \([^)]+\)/i, "IN ('[REDACTED]')"),
|
154
|
+
Redactor.new(/(\s|\b|`)(=|!=|>=|>|<=|<) ?(BINARY )?-?\d+(\.\d+)?/i, " = '[REDACTED]'"),
|
154
155
|
Redactor.new(/(\s|\b|`)(=|!=|>=|>|<=|<) ?(BINARY )?x?#{quotedValuePattern}/i, " = '[REDACTED]'"),
|
155
|
-
Redactor.new(/VALUES \(.+\)$/i, "VALUES ([REDACTED])"),
|
156
|
-
Redactor.new(/IN \([^)]+\)/i, "IN ([REDACTED])"),
|
156
|
+
Redactor.new(/VALUES \(.+\)$/i, "VALUES ('[REDACTED]')"),
|
157
157
|
Redactor.new(/BETWEEN #{quotedValuePattern} AND #{quotedValuePattern}/i, "BETWEEN '[REDACTED]' AND '[REDACTED]'"),
|
158
|
-
Redactor.new(/LIKE #{quotedValuePattern}/i, "LIKE '[REDACTED]'"),
|
158
|
+
Redactor.new(/LIKE #{quotedValuePattern}/i, "LIKE '[REDACTED]'"),
|
159
159
|
Redactor.new(/ LIMIT \d+/i, ""),
|
160
160
|
Redactor.new(/ OFFSET \d+/i, ""),
|
161
|
-
Redactor.new(/INSERT INTO (`?\w+`?) \([^)]+\)/i,
|
161
|
+
Redactor.new(/INSERT INTO (`?\w+`?) \([^)]+\)/i, "INSERT INTO \\1 (REDACTED_COLUMNS)"),
|
162
162
|
]
|
163
163
|
|
164
164
|
@options[:should_log_sample_proc] = Proc.new { |_name| false }
|
@@ -1,3 +1,4 @@
|
|
1
|
+
require 'securerandom'
|
1
2
|
|
2
3
|
module ActiveRecord
|
3
4
|
module SqlAnalyzer
|
@@ -9,7 +10,7 @@ module ActiveRecord
|
|
9
10
|
safe_sql = nil
|
10
11
|
|
11
12
|
SqlAnalyzer.config[:analyzers].each do |analyzer|
|
12
|
-
if
|
13
|
+
if _query_analyzer_private_should_sample_query(analyzer[:name])
|
13
14
|
# This is here rather than above intentionally.
|
14
15
|
# We assume we're not going to be analyzing 100% of queries and want to only re-encode
|
15
16
|
# when it's actually relevant.
|
@@ -21,7 +22,8 @@ module ActiveRecord
|
|
21
22
|
caller: caller,
|
22
23
|
logger: analyzer[:logger_instance],
|
23
24
|
tag: Thread.current[:_ar_analyzer_tag],
|
24
|
-
request_path: Thread.current[:_ar_analyzer_request_path]
|
25
|
+
request_path: Thread.current[:_ar_analyzer_request_path],
|
26
|
+
transaction: @_query_analyzer_private_transaction_uuid
|
25
27
|
}
|
26
28
|
end
|
27
29
|
end
|
@@ -29,6 +31,42 @@ module ActiveRecord
|
|
29
31
|
|
30
32
|
super
|
31
33
|
end
|
34
|
+
|
35
|
+
# Whether or not we should sample the query. If it's part of a transaction, check whether we
|
36
|
+
# are sampling the transaction. Otherwise, run the sample proc.
|
37
|
+
def _query_analyzer_private_should_sample_query(analyzer_name)
|
38
|
+
if @_query_analyzer_private_transaction_uuid.present?
|
39
|
+
@_query_analyzer_private_should_sample_transaction[analyzer_name]
|
40
|
+
else
|
41
|
+
SqlAnalyzer.config[:should_log_sample_proc].call(analyzer_name)
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
# Calculate hash of analyzer_name -> boolean representing whether we should sample the
|
46
|
+
# whole transaction for this analyzer.
|
47
|
+
def _query_analyzer_private_calculate_sampling_for_all_analyzers
|
48
|
+
SqlAnalyzer.config[:analyzers].map do |analyzer|
|
49
|
+
[analyzer[:name], SqlAnalyzer.config[:should_log_sample_proc].call(analyzer[:name])]
|
50
|
+
end.to_h
|
51
|
+
end
|
52
|
+
|
53
|
+
def transaction(requires_new: nil, isolation: nil, joinable: true)
|
54
|
+
must_clear_uuid = false
|
55
|
+
|
56
|
+
if @_query_analyzer_private_transaction_uuid.nil? then
|
57
|
+
must_clear_uuid = true
|
58
|
+
@_query_analyzer_private_transaction_uuid = SecureRandom.uuid
|
59
|
+
@_query_analyzer_private_should_sample_transaction =
|
60
|
+
_query_analyzer_private_calculate_sampling_for_all_analyzers
|
61
|
+
end
|
62
|
+
|
63
|
+
super
|
64
|
+
ensure
|
65
|
+
if must_clear_uuid
|
66
|
+
@_query_analyzer_private_transaction_uuid = nil
|
67
|
+
@_query_analyzer_private_should_sample_transaction = nil
|
68
|
+
end
|
69
|
+
end
|
32
70
|
end
|
33
71
|
end
|
34
72
|
end
|
@@ -74,7 +74,7 @@ RSpec.describe "End to End" do
|
|
74
74
|
|
75
75
|
expect(log_def_hash.length).to eq(2)
|
76
76
|
|
77
|
-
expect(log_def_hash[id_sha]["sql"]).to include("mt.id = [REDACTED]")
|
77
|
+
expect(log_def_hash[id_sha]["sql"]).to include("mt.id = '[REDACTED]'")
|
78
78
|
expect(log_def_hash[str_sha]["sql"]).to include("mt.test_string = '[REDACTED]'")
|
79
79
|
end
|
80
80
|
|
@@ -90,7 +90,7 @@ RSpec.describe "End to End" do
|
|
90
90
|
|
91
91
|
expect(log_def_hash.length).to eq(2)
|
92
92
|
|
93
|
-
expect(log_def_hash[id_sha]["sql"]).to include("id = [REDACTED]")
|
93
|
+
expect(log_def_hash[id_sha]["sql"]).to include("id = '[REDACTED]'")
|
94
94
|
expect(log_def_hash[str_sha]["sql"]).to include("test_string = '[REDACTED]'")
|
95
95
|
end
|
96
96
|
|
@@ -104,6 +104,102 @@ RSpec.describe "End to End" do
|
|
104
104
|
expect(log_def_hash[sha]["sql"]).to include("test_string = '[REDACTED]'")
|
105
105
|
end
|
106
106
|
|
107
|
+
it "logs multiple queries in a transaction correctly" do
|
108
|
+
DBConnection.connection.transaction do
|
109
|
+
4.times { execute "SELECT * FROM matching_table WHERE id = 4321" }
|
110
|
+
3.times { execute "SELECT * FROM matching_table WHERE test_string = 'abc'" }
|
111
|
+
end
|
112
|
+
|
113
|
+
DBConnection.connection.transaction do
|
114
|
+
2.times { execute "SELECT * FROM matching_table WHERE test_string LIKE 'abc'" }
|
115
|
+
execute "SELECT * FROM matching_table WHERE id > 2 and id < 4"
|
116
|
+
end
|
117
|
+
|
118
|
+
id_eq_sha = log_reverse_hash[4]
|
119
|
+
str_eq_sha = log_reverse_hash[3]
|
120
|
+
str_like_sha = log_reverse_hash[2]
|
121
|
+
id_gt_sha = log_reverse_hash[1]
|
122
|
+
|
123
|
+
expect(log_def_hash[id_gt_sha]["sql"]).to include("id = [REDACTED] and id = [REDACTED]")
|
124
|
+
expect(log_def_hash[str_like_sha]["sql"]).to include("test_string LIKE '[REDACTED]'")
|
125
|
+
expect(log_def_hash[id_gt_sha]["transaction"]).to eq(log_def_hash[str_like_sha]["transaction"])
|
126
|
+
|
127
|
+
expect(log_def_hash[id_eq_sha]["sql"]).to include("id = [REDACTED]")
|
128
|
+
expect(log_def_hash[str_eq_sha]["sql"]).to include("test_string = '[REDACTED]'")
|
129
|
+
expect(log_def_hash[str_eq_sha]["transaction"]).to eq(log_def_hash[id_eq_sha]["transaction"])
|
130
|
+
|
131
|
+
expect(log_def_hash[str_like_sha]["transaction"]).to_not eq(log_def_hash[id_eq_sha]["transaction"])
|
132
|
+
end
|
133
|
+
|
134
|
+
it "Logs nested transactions correctly" do
|
135
|
+
DBConnection.connection.transaction do
|
136
|
+
execute "SELECT * FROM matching_table WHERE id = 4321"
|
137
|
+
DBConnection.connection.transaction do
|
138
|
+
2.times { execute "SELECT * FROM matching_table WHERE test_string = 'abc'" }
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
|
143
|
+
id_sha = log_reverse_hash[1]
|
144
|
+
str_sha = log_reverse_hash[2]
|
145
|
+
|
146
|
+
expect(log_def_hash[id_sha]["transaction"]).to eq(log_def_hash[str_sha]["transaction"])
|
147
|
+
expect(log_def_hash[str_sha]["transaction"]).not_to be_nil
|
148
|
+
end
|
149
|
+
|
150
|
+
it "Logs empty transactions correctly" do
|
151
|
+
DBConnection.connection.transaction do
|
152
|
+
execute "SELECT * FROM matching_table WHERE id = 4321"
|
153
|
+
end
|
154
|
+
|
155
|
+
2.times { execute "SELECT * FROM matching_table WHERE test_string = 'abc'" }
|
156
|
+
|
157
|
+
id_sha = log_reverse_hash[1]
|
158
|
+
str_sha = log_reverse_hash[2]
|
159
|
+
|
160
|
+
expect(log_def_hash[id_sha]["sql"]).to include("id = [REDACTED]")
|
161
|
+
expect(log_def_hash[str_sha]["sql"]).to include("test_string = '[REDACTED]'")
|
162
|
+
|
163
|
+
expect(log_def_hash[id_sha]["transaction"]).not_to be_nil
|
164
|
+
expect(log_def_hash[str_sha]["transaction"]).to be_nil
|
165
|
+
end
|
166
|
+
|
167
|
+
context "Selectively sampling" do
|
168
|
+
before do
|
169
|
+
ActiveRecord::SqlAnalyzer.configure do |c|
|
170
|
+
times_called = 0
|
171
|
+
# Return true every other call, starting with the first call
|
172
|
+
c.log_sample_proc Proc.new { |_name| (times_called += 1) % 2 == 1 }
|
173
|
+
end
|
174
|
+
end
|
175
|
+
|
176
|
+
it "Samples some but not other selects" do
|
177
|
+
execute "SELECT * FROM matching_table WHERE id = 1"
|
178
|
+
execute "SELECT * FROM matching_table WHERE test_string = 'abc'"
|
179
|
+
|
180
|
+
expect(log_def_hash.size).to eq(1)
|
181
|
+
expect(log_def_hash.map { |_hash, query| query['sql'] }).to eq([
|
182
|
+
"SELECT * FROM matching_table WHERE id = [REDACTED]"])
|
183
|
+
end
|
184
|
+
|
185
|
+
it "Samples some but not other whole transactions" do
|
186
|
+
DBConnection.connection.transaction do
|
187
|
+
execute "SELECT * FROM matching_table WHERE id = 1"
|
188
|
+
execute "SELECT * FROM matching_table WHERE test_string = 'abc'"
|
189
|
+
end
|
190
|
+
|
191
|
+
DBConnection.connection.transaction do
|
192
|
+
execute "SELECT * FROM matching_table WHERE id = 1 and test_string = 'abc'"
|
193
|
+
execute "SELECT * FROM matching_table WHERE id > 4 and id < 8"
|
194
|
+
end
|
195
|
+
|
196
|
+
expect(log_def_hash.size).to eq(2)
|
197
|
+
expect(Set.new(log_def_hash.map { |_hash, query| query['sql'] })).to eq(Set.new([
|
198
|
+
"SELECT * FROM matching_table WHERE id = [REDACTED]",
|
199
|
+
"SELECT * FROM matching_table WHERE test_string = '[REDACTED]'"]))
|
200
|
+
end
|
201
|
+
end
|
202
|
+
|
107
203
|
context "when sampling is disabled" do
|
108
204
|
before do
|
109
205
|
ActiveRecord::SqlAnalyzer.configure do |c|
|
@@ -1,5 +1,9 @@
|
|
1
|
+
require "spec_helper"
|
2
|
+
require "sql-parser"
|
3
|
+
|
1
4
|
RSpec.describe ActiveRecord::SqlAnalyzer::RedactedLogger do
|
2
5
|
let(:tmp_dir) { Dir.mktmpdir }
|
6
|
+
let(:parser) { SQLParser::Parser.new }
|
3
7
|
after { FileUtils.remove_entry(tmp_dir) }
|
4
8
|
|
5
9
|
context "#filter_event" do
|
@@ -17,6 +21,13 @@ RSpec.describe ActiveRecord::SqlAnalyzer::RedactedLogger do
|
|
17
21
|
[/erb_erb_[0-9]+_[0-9]+/, ""]
|
18
22
|
]
|
19
23
|
end
|
24
|
+
# All raw SQL should be valid :)
|
25
|
+
expect { parser.scan_str(event[:sql]) }.not_to raise_exception if event[:sql].present?
|
26
|
+
end
|
27
|
+
|
28
|
+
after do
|
29
|
+
# All redacted SQL should be valid
|
30
|
+
expect { parser.scan_str(filter_event[:sql]) }.not_to raise_exception if filter_event[:sql].present?
|
20
31
|
end
|
21
32
|
|
22
33
|
context "ambiguous backtraces" do
|
@@ -101,7 +112,7 @@ RSpec.describe ActiveRecord::SqlAnalyzer::RedactedLogger do
|
|
101
112
|
|
102
113
|
it "redacts" do
|
103
114
|
expect(filter_event[:sql]).to eq("SELECT * FROM foo WHERE name LIKE '[REDACTED]'")
|
104
|
-
end
|
115
|
+
end
|
105
116
|
end
|
106
117
|
|
107
118
|
context "sql" do
|
@@ -113,7 +124,7 @@ RSpec.describe ActiveRecord::SqlAnalyzer::RedactedLogger do
|
|
113
124
|
end
|
114
125
|
|
115
126
|
it "redacts" do
|
116
|
-
expect(filter_event[:sql]).to eq("SELECT * FROM foo WHERE id = [REDACTED]")
|
127
|
+
expect(filter_event[:sql]).to eq("SELECT * FROM foo WHERE id = '[REDACTED]'")
|
117
128
|
end
|
118
129
|
end
|
119
130
|
|
@@ -121,12 +132,12 @@ RSpec.describe ActiveRecord::SqlAnalyzer::RedactedLogger do
|
|
121
132
|
let(:event) do
|
122
133
|
{
|
123
134
|
caller: [""],
|
124
|
-
sql:
|
135
|
+
sql: %{SELECT * FROM foo WHERE name LIKE 'A \\'quoted\\' value.' OR name LIKE "another ""quoted"" \\"value\\""}
|
125
136
|
}
|
126
137
|
end
|
127
138
|
|
128
139
|
it "redacts" do
|
129
|
-
expect(filter_event[:sql]).to eq("SELECT * FROM foo WHERE name LIKE '[REDACTED]'")
|
140
|
+
expect(filter_event[:sql]).to eq("SELECT * FROM foo WHERE name LIKE '[REDACTED]' OR name LIKE '[REDACTED]'")
|
130
141
|
end
|
131
142
|
end
|
132
143
|
|
@@ -147,12 +158,12 @@ RSpec.describe ActiveRecord::SqlAnalyzer::RedactedLogger do
|
|
147
158
|
let(:event) do
|
148
159
|
{
|
149
160
|
caller: [""],
|
150
|
-
sql: "SELECT * FROM foo WHERE name IN ('A
|
161
|
+
sql: "SELECT * FROM foo WHERE name IN ('A ''quoted'' value.')"
|
151
162
|
}
|
152
163
|
end
|
153
164
|
|
154
165
|
it "redacts" do
|
155
|
-
expect(filter_event[:sql]).to eq("SELECT * FROM foo WHERE name IN ([REDACTED])")
|
166
|
+
expect(filter_event[:sql]).to eq("SELECT * FROM foo WHERE name IN ('[REDACTED]')")
|
156
167
|
end
|
157
168
|
end
|
158
169
|
|
@@ -160,20 +171,20 @@ RSpec.describe ActiveRecord::SqlAnalyzer::RedactedLogger do
|
|
160
171
|
let(:event) do
|
161
172
|
{
|
162
173
|
caller: [""],
|
163
|
-
sql:
|
174
|
+
sql: %{SELECT * FROM foo WHERE name IN ('A ''quoted'' value.', "another ""quoted"" \\"value\\"")}
|
164
175
|
}
|
165
176
|
end
|
166
177
|
|
167
178
|
it "redacts" do
|
168
|
-
expect(filter_event[:sql]).to eq("SELECT * FROM foo WHERE name IN ([REDACTED])")
|
179
|
+
expect(filter_event[:sql]).to eq("SELECT * FROM foo WHERE name IN ('[REDACTED]')")
|
169
180
|
end
|
170
181
|
end
|
171
182
|
|
172
|
-
context "between
|
183
|
+
context "between strings" do
|
173
184
|
let(:event) do
|
174
185
|
{
|
175
186
|
caller: [""],
|
176
|
-
sql: "SELECT * FROM foo WHERE name BETWEEN 'A
|
187
|
+
sql: "SELECT * FROM foo WHERE name BETWEEN 'A value.' AND 'Another value'"
|
177
188
|
}
|
178
189
|
end
|
179
190
|
|
@@ -182,11 +193,11 @@ RSpec.describe ActiveRecord::SqlAnalyzer::RedactedLogger do
|
|
182
193
|
end
|
183
194
|
end
|
184
195
|
|
185
|
-
context "between escaped
|
196
|
+
context "between strings with escaped quotes" do
|
186
197
|
let(:event) do
|
187
198
|
{
|
188
199
|
caller: [""],
|
189
|
-
sql: "SELECT * FROM foo WHERE name BETWEEN 'A
|
200
|
+
sql: "SELECT * FROM foo WHERE name BETWEEN 'A ''quoted'' value.' AND 'Another \\'value\\''"
|
190
201
|
}
|
191
202
|
end
|
192
203
|
|
@@ -194,5 +205,31 @@ RSpec.describe ActiveRecord::SqlAnalyzer::RedactedLogger do
|
|
194
205
|
expect(filter_event[:sql]).to eq("SELECT * FROM foo WHERE name BETWEEN '[REDACTED]' AND '[REDACTED]'")
|
195
206
|
end
|
196
207
|
end
|
208
|
+
|
209
|
+
context "in with = and other where clauses" do
|
210
|
+
let(:event) do
|
211
|
+
{
|
212
|
+
caller: [""],
|
213
|
+
sql: "SELECT * FROM foo WHERE name IN ('value=') AND name = 'value'"
|
214
|
+
}
|
215
|
+
end
|
216
|
+
|
217
|
+
it "redacts" do
|
218
|
+
expect(filter_event[:sql]).to eq("SELECT * FROM foo WHERE name IN ('[REDACTED]') AND name = '[REDACTED]'")
|
219
|
+
end
|
220
|
+
end
|
221
|
+
|
222
|
+
context "insert" do
|
223
|
+
let(:event) do
|
224
|
+
{
|
225
|
+
caller: [""],
|
226
|
+
sql: "INSERT INTO `boom` (`bam`, `foo`) VALUES ('howdy', 'dowdy')"
|
227
|
+
}
|
228
|
+
end
|
229
|
+
|
230
|
+
it "redacts" do
|
231
|
+
expect(filter_event[:sql]).to eq("INSERT INTO `boom` (REDACTED_COLUMNS) VALUES ('[REDACTED]')")
|
232
|
+
end
|
233
|
+
end
|
197
234
|
end
|
198
235
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: active_record-sql_analyzer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Zachary Anker
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2016-
|
12
|
+
date: 2016-08-09 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: activerecord
|
@@ -110,7 +110,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
110
110
|
version: '0'
|
111
111
|
requirements: []
|
112
112
|
rubyforge_project:
|
113
|
-
rubygems_version: 2.
|
113
|
+
rubygems_version: 2.5.1
|
114
114
|
signing_key:
|
115
115
|
specification_version: 4
|
116
116
|
summary: Logs a subset of ActiveRecord queries and dumps them for analyses.
|
@@ -127,4 +127,3 @@ test_files:
|
|
127
127
|
- spec/support/stub_logger.rb
|
128
128
|
- spec/support/stub_rails.rb
|
129
129
|
- spec/support/wait_for_pop.rb
|
130
|
-
has_rdoc:
|