scout_apm 6.3.2 → 6.3.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/semgrep.yml +41 -0
- data/CHANGELOG.markdown +6 -0
- data/lib/scout_apm/serializers/payload_serializer_to_json.rb +20 -36
- data/lib/scout_apm/utils/sql_sanitizer.rb +14 -13
- data/lib/scout_apm/version.rb +1 -1
- data/test/unit/serializers/payload_serializer_test.rb +26 -0
- data/test/unit/sql_sanitizer_test.rb +53 -0
- metadata +2 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 698ac525d5b73cd4138f52ebb08fa149d8d6f272057b3821a9104f10ac502868
|
|
4
|
+
data.tar.gz: b1ba437be29db44c25c9a1b725ea8c6dd902f4b6cd13bd359fb9333665e7faad
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: c36c0715f438b80da07a50a0dba8f47452e6b446a2bb747b34348e86199fd3909ce4387c7228c016c25645cf42d69f132acbc1455b49489d6400653deafc91c0
|
|
7
|
+
data.tar.gz: 826b8c2b2c6b4e911fc732dd1de09f0045becd8857ce163c01bf47bad2d7a161c555593d08f69d9247bfec5c82be5310bbef89c664905a2b87b554320515ee9b
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
name: Semgrep
|
|
2
|
+
|
|
3
|
+
# Static analysis (Semgrep) on every PR to master. PRs are diff-aware: only findings
|
|
4
|
+
# a PR introduces fail the check. Pushes to master run a full, report-only scan.
|
|
5
|
+
|
|
6
|
+
on:
|
|
7
|
+
pull_request:
|
|
8
|
+
branches:
|
|
9
|
+
- master
|
|
10
|
+
push:
|
|
11
|
+
branches:
|
|
12
|
+
- master
|
|
13
|
+
workflow_dispatch:
|
|
14
|
+
|
|
15
|
+
permissions:
|
|
16
|
+
contents: read
|
|
17
|
+
|
|
18
|
+
jobs:
|
|
19
|
+
semgrep:
|
|
20
|
+
name: Semgrep SAST
|
|
21
|
+
runs-on: ubuntu-latest
|
|
22
|
+
timeout-minutes: 15
|
|
23
|
+
container:
|
|
24
|
+
image: semgrep/semgrep:1.183.0
|
|
25
|
+
|
|
26
|
+
steps:
|
|
27
|
+
- name: Checkout code
|
|
28
|
+
uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
|
|
29
|
+
with:
|
|
30
|
+
fetch-depth: 0
|
|
31
|
+
|
|
32
|
+
- name: Scan
|
|
33
|
+
env:
|
|
34
|
+
BASE_SHA: ${{ github.event.pull_request.base.sha }}
|
|
35
|
+
run: |
|
|
36
|
+
git config --global --add safe.directory "$GITHUB_WORKSPACE"
|
|
37
|
+
if [ -n "$BASE_SHA" ]; then
|
|
38
|
+
semgrep scan --config p/default --metrics=off --error --baseline-commit "$BASE_SHA"
|
|
39
|
+
else
|
|
40
|
+
semgrep scan --config p/default --metrics=off
|
|
41
|
+
fi
|
data/CHANGELOG.markdown
CHANGED
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
# Pending
|
|
2
2
|
|
|
3
|
+
# 6.3.3
|
|
4
|
+
|
|
5
|
+
- Encode the JSON reporting payload with the json library instead of the hand-written encoder (#641)
|
|
6
|
+
- Sanitize each SQL statement only once instead of on every `SqlSanitizer#to_s` call (#642)
|
|
7
|
+
- Remove `[[...]]` bind lists that are on their own lines from Postgres SQL (#645)
|
|
8
|
+
|
|
3
9
|
# 6.3.2
|
|
4
10
|
|
|
5
11
|
- Build the per-minute debug dumps of metrics, slow traces and auto-instrument histograms only when debug logging is enabled (#639)
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
require 'json'
|
|
2
|
+
|
|
1
3
|
module ScoutApm
|
|
2
4
|
module Serializers
|
|
3
5
|
module PayloadSerializerToJson
|
|
@@ -39,50 +41,32 @@ module ScoutApm
|
|
|
39
41
|
end
|
|
40
42
|
|
|
41
43
|
def jsonify_hash(hash)
|
|
42
|
-
|
|
43
|
-
hash.each do |key, value|
|
|
44
|
-
formatted_key = format_by_type(key)
|
|
45
|
-
formatted_value = format_by_type(value)
|
|
46
|
-
str_parts << "#{formatted_key}:#{formatted_value}"
|
|
47
|
-
end
|
|
48
|
-
"{#{str_parts.join(",")}}"
|
|
49
|
-
end
|
|
50
|
-
|
|
51
|
-
ESCAPE_MAPPINGS = {
|
|
52
|
-
# Stackoverflow answer on gsub matches and backslashes
|
|
53
|
-
# https://stackoverflow.com/a/4149087/2705125
|
|
54
|
-
'\\' => '\\\\\\\\',
|
|
55
|
-
"\b" => '\\b',
|
|
56
|
-
"\t" => '\\t',
|
|
57
|
-
"\n" => '\\n',
|
|
58
|
-
"\f" => '\\f',
|
|
59
|
-
"\r" => '\\r',
|
|
60
|
-
'"' => '\\"',
|
|
61
|
-
}
|
|
62
|
-
|
|
63
|
-
def escape(string)
|
|
64
|
-
ESCAPE_MAPPINGS.inject(string.to_s) {|s, (bad, good)|
|
|
65
|
-
s.gsub(bad, good)
|
|
66
|
-
}
|
|
44
|
+
JSON.generate(json_ready(hash), :allow_nan => true, :max_nesting => false)
|
|
67
45
|
end
|
|
68
46
|
|
|
69
|
-
def
|
|
70
|
-
case
|
|
47
|
+
def json_ready(value)
|
|
48
|
+
case value
|
|
71
49
|
when Hash
|
|
72
|
-
|
|
50
|
+
value.each_with_object({}) { |(key, item), ready| ready[json_ready(key)] = json_ready(item) }
|
|
73
51
|
when Array
|
|
74
|
-
|
|
75
|
-
|
|
52
|
+
value.map { |item| json_ready(item) }
|
|
53
|
+
when Integer, Float, nil
|
|
54
|
+
value
|
|
76
55
|
when Numeric
|
|
77
|
-
|
|
56
|
+
value.to_f
|
|
78
57
|
when Time
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
else # strings and everything
|
|
83
|
-
%Q["#{escape(formatee)}"]
|
|
58
|
+
value.iso8601
|
|
59
|
+
else
|
|
60
|
+
valid_utf8(value.to_s)
|
|
84
61
|
end
|
|
85
62
|
end
|
|
63
|
+
|
|
64
|
+
def valid_utf8(string)
|
|
65
|
+
return string if string.valid_encoding? && string.encoding != Encoding::BINARY
|
|
66
|
+
|
|
67
|
+
string = string.dup.force_encoding(Encoding::UTF_8)
|
|
68
|
+
string.valid_encoding? ? string : string.scrub
|
|
69
|
+
end
|
|
86
70
|
end
|
|
87
71
|
end
|
|
88
72
|
end
|
|
@@ -11,6 +11,10 @@ module ScoutApm
|
|
|
11
11
|
MULTIPLE_QUESTIONS = /\?(,\?)+/.freeze
|
|
12
12
|
|
|
13
13
|
PSQL_VAR_INTERPOLATION = %r|\[\[.*\]\]\s*\z|.freeze
|
|
14
|
+
# A line holding only a bind list, optionally followed by /* */ comments.
|
|
15
|
+
# The list body can't run past a "/*", and a comment body stops at its
|
|
16
|
+
# first "*/", so each line is matched in linear time.
|
|
17
|
+
PSQL_VAR_INTERPOLATION_LINES = %r{^[ \t]*\[\[(?:[^/\n]|/(?!\*))*\]\](?=(?:[ \t]*/\*(?:[^*\n]|\*(?!/))*\*/)*[ \t\r]*$)}.freeze
|
|
14
18
|
PSQL_REMOVE_STRINGS = /'(?:[^']|'')*'/.freeze
|
|
15
19
|
PSQL_REMOVE_JSON_STRINGS = /:"(?:[^"]|"")*"/.freeze
|
|
16
20
|
PSQL_REMOVE_INTEGERS = /(?<!LIMIT )\b\d+\b/.freeze
|
|
@@ -44,7 +48,6 @@ module ScoutApm
|
|
|
44
48
|
def initialize(sql)
|
|
45
49
|
@raw_sql = sql
|
|
46
50
|
@database_engine = ScoutApm::Agent.instance.context.environment.database_engine
|
|
47
|
-
@sanitized = false # only sanitize once.
|
|
48
51
|
end
|
|
49
52
|
|
|
50
53
|
def sql
|
|
@@ -52,17 +55,12 @@ module ScoutApm
|
|
|
52
55
|
end
|
|
53
56
|
|
|
54
57
|
def to_s
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
when :postgres then to_s_postgres
|
|
62
|
-
when :mysql then to_s_mysql
|
|
63
|
-
when :sqlite then to_s_sqlite
|
|
64
|
-
when :sqlserver then to_s_sqlserver
|
|
65
|
-
end
|
|
58
|
+
@to_s ||= case database_engine
|
|
59
|
+
when :postgres then to_s_postgres
|
|
60
|
+
when :mysql then to_s_mysql
|
|
61
|
+
when :sqlite then to_s_sqlite
|
|
62
|
+
when :sqlserver then to_s_sqlserver
|
|
63
|
+
end
|
|
66
64
|
end
|
|
67
65
|
|
|
68
66
|
private
|
|
@@ -77,7 +75,10 @@ module ScoutApm
|
|
|
77
75
|
|
|
78
76
|
def to_s_postgres
|
|
79
77
|
sql.gsub!(PSQL_PLACEHOLDER, '?')
|
|
80
|
-
sql.
|
|
78
|
+
if sql.include?('[[')
|
|
79
|
+
sql.gsub!(PSQL_VAR_INTERPOLATION, '')
|
|
80
|
+
sql.gsub!(PSQL_VAR_INTERPOLATION_LINES, '')
|
|
81
|
+
end
|
|
81
82
|
# sql.gsub!(PSQL_REMOVE_STRINGS, '?')
|
|
82
83
|
sql.gsub!(PSQL_AFTER_WHERE) {|c| c.gsub(PSQL_REMOVE_STRINGS, '?')}
|
|
83
84
|
sql.gsub!(PSQL_AFTER_FROM_AS) {|c| c.gsub(PSQL_REMOVE_JSON_STRINGS, ':"?"')}
|
data/lib/scout_apm/version.rb
CHANGED
|
@@ -144,4 +144,30 @@ class PayloadSerializerTest < Minitest::Test
|
|
|
144
144
|
result = ScoutApm::Serializers::PayloadSerializerToJson.jsonify_hash(json)
|
|
145
145
|
assert_equal json, JSON.parse(result)
|
|
146
146
|
end
|
|
147
|
+
|
|
148
|
+
def test_encodes_non_string_values
|
|
149
|
+
hash = {
|
|
150
|
+
:time => Time.utc(2026, 10, 29, 10, 0, 1),
|
|
151
|
+
:literals => [nil, true, false, :symbol],
|
|
152
|
+
:numbers => [1, 0.5, Rational(1, 4)],
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
result = JSON.parse(ScoutApm::Serializers::PayloadSerializerToJson.jsonify_hash(hash))
|
|
156
|
+
|
|
157
|
+
assert_equal({ "time" => "2026-10-29T10:00:01Z", "literals" => [nil, "true", "false", "symbol"], "numbers" => [1, 0.5, 0.25] }, result)
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
def test_keeps_non_finite_floats
|
|
161
|
+
result = ScoutApm::Serializers::PayloadSerializerToJson.jsonify_hash({ "nan" => Float::NAN, "inf" => Float::INFINITY })
|
|
162
|
+
|
|
163
|
+
assert_equal '{"nan":NaN,"inf":Infinity}', result
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def test_replaces_invalid_utf8_bytes
|
|
167
|
+
hash = { "utf8" => "bad \xFF byte".dup.force_encoding("UTF-8"), "binary" => "caf\xC3\xA9 \xFF".b }
|
|
168
|
+
|
|
169
|
+
result = ScoutApm::Serializers::PayloadSerializerToJson.jsonify_hash(hash)
|
|
170
|
+
|
|
171
|
+
assert_equal({ "utf8" => "bad \uFFFD byte", "binary" => "caf\u00E9 \uFFFD" }, JSON.parse(result))
|
|
172
|
+
end
|
|
147
173
|
end
|
|
@@ -25,6 +25,50 @@ module ScoutApm
|
|
|
25
25
|
assert_equal %q|SELECT "users".* FROM "users" WHERE "users"."name" = ?|, ss.to_s
|
|
26
26
|
end
|
|
27
27
|
|
|
28
|
+
def test_postgres_where_multiline_binds
|
|
29
|
+
sql = %Q|SELECT "users".* FROM "users" WHERE "users"."name" = $1 AND "users"."id" = $2\n [["name", "chris"]]\n [["id", 1]]|
|
|
30
|
+
ss = SqlSanitizer.new(sql).tap{ |it| it.database_engine = :postgres }
|
|
31
|
+
assert_equal %q|SELECT "users".* FROM "users" WHERE "users"."name" = ? AND "users"."id" = ?|, ss.to_s
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def test_postgres_where_multiline_binds_crlf
|
|
35
|
+
sql = %Q|SELECT "users".* FROM "users" WHERE "users"."name" = $1 AND "users"."id" = $2\r\n [["name", "chris"]]\r\n [["id", 1]]|
|
|
36
|
+
ss = SqlSanitizer.new(sql).tap{ |it| it.database_engine = :postgres }
|
|
37
|
+
assert_equal %q|SELECT "users".* FROM "users" WHERE "users"."name" = ? AND "users"."id" = ?|, ss.to_s
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def test_postgres_multiline_binds_before_comment
|
|
41
|
+
sql = %Q|SELECT "users".* FROM "users" WHERE "users"."name" = $1\n [["name", "chris"]]\n /*application:Blog*/|
|
|
42
|
+
ss = SqlSanitizer.new(sql).tap{ |it| it.database_engine = :postgres }
|
|
43
|
+
assert_equal %q|SELECT "users".* FROM "users" WHERE "users"."name" = ? /*application:Blog*/|, ss.to_s
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def test_postgres_multiline_binds_with_comment_on_same_line
|
|
47
|
+
sql = %Q|SELECT "users".* FROM "users" WHERE "users"."name" = $1 AND "users"."id" = $2\n [["id", 1]]\n [["name", "chris"]] /*application:Blog*/|
|
|
48
|
+
ss = SqlSanitizer.new(sql).tap{ |it| it.database_engine = :postgres }
|
|
49
|
+
assert_equal %q|SELECT "users".* FROM "users" WHERE "users"."name" = ? AND "users"."id" = ? /*application:Blog*/|, ss.to_s
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def test_postgres_keeps_array_literal_before_multiline_binds
|
|
53
|
+
sql = %Q|SELECT "users".* FROM "users" WHERE "users"."tags" = ARRAY[[1,2],[3,4]]\n [["id", 1]]|
|
|
54
|
+
ss = SqlSanitizer.new(sql).tap{ |it| it.database_engine = :postgres }
|
|
55
|
+
assert_equal %q|SELECT "users".* FROM "users" WHERE "users"."tags" = ARRAY[[?,?],[?,?]]|, ss.to_s
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# Inputs that make an unanchored, repeated or unbounded version of the
|
|
59
|
+
# bind-list line pattern backtrack quadratically or worse.
|
|
60
|
+
def test_postgres_bind_list_line_pattern_is_linear
|
|
61
|
+
[
|
|
62
|
+
"SELECT 1 " + ("[[" * 7000),
|
|
63
|
+
("[[1]]\n" * 2700) + "x",
|
|
64
|
+
"[[" + ("]] /* x" * 2000),
|
|
65
|
+
].each do |sql|
|
|
66
|
+
assert_faster_than(0.03) do
|
|
67
|
+
sql.gsub(SqlSanitizer::PSQL_VAR_INTERPOLATION_LINES, '')
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
|
|
28
72
|
def test_postgres_strips_literals
|
|
29
73
|
# Strip strings
|
|
30
74
|
sql = %q|SELECT "users".* FROM "users" INNER JOIN "blogs" ON "blogs"."user_id" = "users"."id" WHERE (blogs.title = 'hello world')|
|
|
@@ -163,6 +207,15 @@ module ScoutApm
|
|
|
163
207
|
assert_equal %q|UPDATE "mytable" SET "myfield" = ?, "countofthings" = ? WHERE "user_id" = ?|, ss.to_s
|
|
164
208
|
end
|
|
165
209
|
|
|
210
|
+
def test_sanitizes_only_once
|
|
211
|
+
sql = %q|SELECT "users".* FROM "users" WHERE "users"."id" = 1|
|
|
212
|
+
ss = SqlSanitizer.new(sql).tap{ |it| it.database_engine = :postgres }
|
|
213
|
+
sanitized = ss.to_s
|
|
214
|
+
|
|
215
|
+
ss.expects(:to_s_postgres).never
|
|
216
|
+
assert_equal sanitized, ss.to_s
|
|
217
|
+
end
|
|
218
|
+
|
|
166
219
|
def test_postgres_multiline_sql
|
|
167
220
|
sql = %q|
|
|
168
221
|
SELECT "html_form_payloads".*
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: scout_apm
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 6.3.
|
|
4
|
+
version: 6.3.3
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Derek Haynes
|
|
@@ -216,6 +216,7 @@ extensions:
|
|
|
216
216
|
extra_rdoc_files: []
|
|
217
217
|
files:
|
|
218
218
|
- ".github/workflows/release.yml"
|
|
219
|
+
- ".github/workflows/semgrep.yml"
|
|
219
220
|
- ".github/workflows/test.yml"
|
|
220
221
|
- ".gitignore"
|
|
221
222
|
- ".rubocop.yml"
|