opentelemetry-helpers-sql-processor 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/lib/opentelemetry/helpers/sql_processor/obfuscator.rb +126 -0
- data/lib/opentelemetry/helpers/sql_processor/version.rb +1 -1
- data/lib/opentelemetry/helpers/sql_processor.rb +36 -0
- data/lib/opentelemetry/helpers.rb +1 -1
- metadata +6 -5
- data/lib/opentelemetry/helpers/sql_obfuscation.rb +0 -130
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 3ed5ecc010586814ebc74f60f2d590eac9d0ab75aaa66aa130ba76f992973d5c
|
|
4
|
+
data.tar.gz: 8d3c91ecb6254424c6e0a9866e34d18edf27dd56981931facc93aef77ef37531
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 244d84fdf356d629f2680cb3c3ece397e7b79b151c7b464fb1415b37ebeaf70421193119b5e7d4584cfc500c2ca392a2c495ba8ee52610c6aaf8310e1d26d3fa
|
|
7
|
+
data.tar.gz: c545cdf4006fdb88281fcf0bfa0d83a46af6e078fbb291a05f343db1aba01e90fa20a6151ad8ffc2d2d080f9d9ee97d88e97bf0ffe84a92cf4fe66029e23731c
|
data/CHANGELOG.md
CHANGED
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Copyright The OpenTelemetry Authors
|
|
4
|
+
#
|
|
5
|
+
# SPDX-License-Identifier: Apache-2.0module OpenTelemetry
|
|
6
|
+
|
|
7
|
+
require 'opentelemetry-common'
|
|
8
|
+
|
|
9
|
+
module OpenTelemetry
|
|
10
|
+
module Helpers
|
|
11
|
+
module SqlProcessor
|
|
12
|
+
#
|
|
13
|
+
# This module contains SQL obfuscation behavior to share with
|
|
14
|
+
# instrumentation for specific database adapters.
|
|
15
|
+
# The class uses code from: https://github.com/newrelic/newrelic-ruby-agent/blob/1fca78cc7a087421ad58088d8bea72c0362bc62f/lib/new_relic/agent/database/obfuscation_helpers.rb
|
|
16
|
+
#
|
|
17
|
+
# To use this in your instrumentation, the `Instrumentation` class for
|
|
18
|
+
# your gem must contain configuration options for:
|
|
19
|
+
# * `:db_statement`
|
|
20
|
+
# Example:
|
|
21
|
+
# `option :db_statement, default: :include, validate: %I[omit include obfuscate]`
|
|
22
|
+
# * `:obfuscation_limit`
|
|
23
|
+
# Example:
|
|
24
|
+
# `option :obfuscation_limit, default: 2000, validate: :integer`
|
|
25
|
+
#
|
|
26
|
+
# If you want to add support for a new adapter, update the following
|
|
27
|
+
# constants to include keys for your adapter:
|
|
28
|
+
# * DIALECT_COMPONENTS
|
|
29
|
+
# * CLEANUP_REGEX
|
|
30
|
+
# You must also add a new constant that uses `generate_regex` with your
|
|
31
|
+
# adapter's dialect components that is named like
|
|
32
|
+
# `<ADAPTER>_COMPONENTS_REGEX`, such as: `MYSQL_COMPONENTS_REGEX`.
|
|
33
|
+
#
|
|
34
|
+
# @api public
|
|
35
|
+
module Obfuscator
|
|
36
|
+
module_function
|
|
37
|
+
|
|
38
|
+
# From: https://github.com/newrelic/newrelic-ruby-agent/blob/1fca78cc7a087421ad58088d8bea72c0362bc62f/lib/new_relic/agent/database/obfuscation_helpers.rb
|
|
39
|
+
COMPONENTS_REGEX_MAP = {
|
|
40
|
+
single_quotes: /'(?:[^']|'')*?(?:\\'.*|'(?!'))/,
|
|
41
|
+
double_quotes: /"(?:[^"]|"")*?(?:\\".*|"(?!"))/,
|
|
42
|
+
dollar_quotes: /(\$(?!\d)[^$]*?\$).*?(?:\1|$)/,
|
|
43
|
+
uuids: /\{?(?:[0-9a-fA-F]\-*){32}\}?/,
|
|
44
|
+
numeric_literals: /-?\b(?:[0-9]+\.)?[0-9]+([eE][+-]?[0-9]+)?\b/,
|
|
45
|
+
boolean_literals: /\b(?:true|false|null)\b/i,
|
|
46
|
+
hexadecimal_literals: /0x[0-9a-fA-F]+/,
|
|
47
|
+
comments: /(?:#|--).*?(?=\r|\n|$)/i,
|
|
48
|
+
multi_line_comments: %r{(?:\/\*.*?\*\/)}m,
|
|
49
|
+
oracle_quoted_strings: /q'\[.*?(?:\]'|$)|q'\{.*?(?:\}'|$)|q'\<.*?(?:\>'|$)|q'\(.*?(?:\)'|$)/
|
|
50
|
+
}.freeze
|
|
51
|
+
|
|
52
|
+
DIALECT_COMPONENTS = {
|
|
53
|
+
default: COMPONENTS_REGEX_MAP.keys,
|
|
54
|
+
mysql: %i[single_quotes double_quotes numeric_literals boolean_literals
|
|
55
|
+
hexadecimal_literals comments multi_line_comments],
|
|
56
|
+
postgres: %i[single_quotes dollar_quotes uuids numeric_literals
|
|
57
|
+
boolean_literals comments multi_line_comments],
|
|
58
|
+
sqlite: %i[single_quotes numeric_literals boolean_literals hexadecimal_literals
|
|
59
|
+
comments multi_line_comments],
|
|
60
|
+
oracle: %i[single_quotes oracle_quoted_strings numeric_literals comments
|
|
61
|
+
multi_line_comments],
|
|
62
|
+
cassandra: %i[single_quotes uuids numeric_literals boolean_literals
|
|
63
|
+
hexadecimal_literals comments multi_line_comments]
|
|
64
|
+
}.freeze
|
|
65
|
+
|
|
66
|
+
PLACEHOLDER = '?'
|
|
67
|
+
|
|
68
|
+
# We use these to check whether the query contains any quote characters
|
|
69
|
+
# after obfuscation. If so, that's a good indication that the original
|
|
70
|
+
# query was malformed, and so our obfuscation can't reliably find
|
|
71
|
+
# literals. In such a case, we'll replace the entire query with a
|
|
72
|
+
# placeholder.
|
|
73
|
+
CLEANUP_REGEX = {
|
|
74
|
+
default: %r{'|"|\/\*|\*\/},
|
|
75
|
+
mysql: %r{'|"|\/\*|\*\//},
|
|
76
|
+
postgres: %r{'|\/\*|\*\/|\$(?!\?)/},
|
|
77
|
+
sqlite: %r{'|\/\*|\*\//},
|
|
78
|
+
cassandra: %r{'|\/\*|\*\//},
|
|
79
|
+
oracle: %r{'|\/\*|\*\//}
|
|
80
|
+
}.freeze
|
|
81
|
+
|
|
82
|
+
# @api private
|
|
83
|
+
def generate_regex(dialect)
|
|
84
|
+
components = DIALECT_COMPONENTS[dialect]
|
|
85
|
+
Regexp.union(components.map { |component| COMPONENTS_REGEX_MAP[component] })
|
|
86
|
+
end
|
|
87
|
+
|
|
88
|
+
DEFAULT_COMPONENTS_REGEX = generate_regex(:default)
|
|
89
|
+
MYSQL_COMPONENTS_REGEX = generate_regex(:mysql)
|
|
90
|
+
POSTGRES_COMPONENTS_REGEX = generate_regex(:postgres)
|
|
91
|
+
SQLITE_COMPONENTS_REGEX = generate_regex(:sqlite)
|
|
92
|
+
CASSANDRA_COMPONENTS_REGEX = generate_regex(:cassandra)
|
|
93
|
+
ORACLE_COMPONENTS_REGEX = generate_regex(:oracle)
|
|
94
|
+
|
|
95
|
+
# Internal implementation of SQL obfuscation.
|
|
96
|
+
# Use SqlProcessor.obfuscate_sql for the public API.
|
|
97
|
+
#
|
|
98
|
+
# @api private
|
|
99
|
+
def obfuscate_sql(sql, obfuscation_limit: 2000, adapter: :default)
|
|
100
|
+
return "SQL not obfuscated, query exceeds #{obfuscation_limit} characters" if sql.size > obfuscation_limit
|
|
101
|
+
|
|
102
|
+
regex = case adapter
|
|
103
|
+
when :mysql
|
|
104
|
+
MYSQL_COMPONENTS_REGEX
|
|
105
|
+
when :postgres
|
|
106
|
+
POSTGRES_COMPONENTS_REGEX
|
|
107
|
+
else
|
|
108
|
+
DEFAULT_COMPONENTS_REGEX
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# Original MySQL UTF-8 Encoding Fixes:
|
|
112
|
+
# https://github.com/open-telemetry/opentelemetry-ruby-contrib/pull/160
|
|
113
|
+
# https://github.com/open-telemetry/opentelemetry-ruby-contrib/pull/345
|
|
114
|
+
sql = OpenTelemetry::Common::Utilities.utf8_encode(sql, binary: true)
|
|
115
|
+
|
|
116
|
+
sql = sql.gsub(regex, PLACEHOLDER)
|
|
117
|
+
return 'Failed to obfuscate SQL query - quote characters remained after obfuscation' if CLEANUP_REGEX[adapter].match(sql)
|
|
118
|
+
|
|
119
|
+
sql
|
|
120
|
+
rescue StandardError => e
|
|
121
|
+
OpenTelemetry.handle_error(message: 'Failed to obfuscate SQL', exception: e)
|
|
122
|
+
end
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
end
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# Copyright The OpenTelemetry Authors
|
|
4
|
+
#
|
|
5
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
6
|
+
|
|
7
|
+
require 'opentelemetry-common'
|
|
8
|
+
require_relative 'sql_processor/obfuscator'
|
|
9
|
+
|
|
10
|
+
module OpenTelemetry
|
|
11
|
+
module Helpers
|
|
12
|
+
# SQL processing utilities for OpenTelemetry instrumentation.
|
|
13
|
+
#
|
|
14
|
+
# This module provides a unified interface for SQL processing operations
|
|
15
|
+
# commonly needed in database adapter instrumentation, including SQL obfuscation.
|
|
16
|
+
#
|
|
17
|
+
# @api public
|
|
18
|
+
module SqlProcessor
|
|
19
|
+
module_function
|
|
20
|
+
|
|
21
|
+
# This is a SQL obfuscation utility intended for use in database adapter instrumentation. It uses the {Obfuscator} module.
|
|
22
|
+
#
|
|
23
|
+
# @param sql [String] The SQL to obfuscate.
|
|
24
|
+
# @param obfuscation_limit [optional Integer] the length at which the SQL string will not be obfuscated
|
|
25
|
+
# @param adapter [optional Symbol] the type of database adapter calling the method. `:default`, `:mysql`, `:postgres`, `:sqlite`, `:oracle`, `:cassandra` are supported.
|
|
26
|
+
# @return [String] The SQL query string where the values are replaced with "?". When the sql statement exceeds the obfuscation limit
|
|
27
|
+
# the first matched pair from the SQL statement will be returned, with an appended truncation message. If truncation is unsuccessful,
|
|
28
|
+
# a string describing the error will be returned.
|
|
29
|
+
#
|
|
30
|
+
# @api public
|
|
31
|
+
def obfuscate_sql(sql, obfuscation_limit: 2000, adapter: :default)
|
|
32
|
+
Obfuscator.obfuscate_sql(sql, obfuscation_limit: obfuscation_limit, adapter: adapter)
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: opentelemetry-helpers-sql-processor
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.3.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- OpenTelemetry Authors
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2025-
|
|
11
|
+
date: 2025-11-04 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: opentelemetry-common
|
|
@@ -37,16 +37,17 @@ files:
|
|
|
37
37
|
- README.md
|
|
38
38
|
- lib/opentelemetry-helpers-sql-processor.rb
|
|
39
39
|
- lib/opentelemetry/helpers.rb
|
|
40
|
-
- lib/opentelemetry/helpers/
|
|
40
|
+
- lib/opentelemetry/helpers/sql_processor.rb
|
|
41
|
+
- lib/opentelemetry/helpers/sql_processor/obfuscator.rb
|
|
41
42
|
- lib/opentelemetry/helpers/sql_processor/version.rb
|
|
42
43
|
homepage: https://github.com/open-telemetry/opentelemetry-ruby-contrib
|
|
43
44
|
licenses:
|
|
44
45
|
- Apache-2.0
|
|
45
46
|
metadata:
|
|
46
|
-
changelog_uri: https://rubydoc.info/gems/opentelemetry-helpers-sql-processor/0.
|
|
47
|
+
changelog_uri: https://rubydoc.info/gems/opentelemetry-helpers-sql-processor/0.3.0/file/CHANGELOG.md
|
|
47
48
|
source_code_uri: https://github.com/open-telemetry/opentelemetry-ruby-contrib/tree/main/helpers/sql-processor
|
|
48
49
|
bug_tracker_uri: https://github.com/open-telemetry/opentelemetry-ruby-contrib/issues
|
|
49
|
-
documentation_uri: https://rubydoc.info/gems/opentelemetry-helpers-sql-processor/0.
|
|
50
|
+
documentation_uri: https://rubydoc.info/gems/opentelemetry-helpers-sql-processor/0.3.0
|
|
50
51
|
post_install_message: ''
|
|
51
52
|
rdoc_options: []
|
|
52
53
|
require_paths:
|
|
@@ -1,130 +0,0 @@
|
|
|
1
|
-
# frozen_string_literal: true
|
|
2
|
-
|
|
3
|
-
# Copyright The OpenTelemetry Authors
|
|
4
|
-
#
|
|
5
|
-
# SPDX-License-Identifier: Apache-2.0module OpenTelemetry
|
|
6
|
-
|
|
7
|
-
require 'opentelemetry-common'
|
|
8
|
-
|
|
9
|
-
module OpenTelemetry
|
|
10
|
-
module Helpers
|
|
11
|
-
#
|
|
12
|
-
# This module contains SQL obfuscation behavior to share with
|
|
13
|
-
# instrumentation for specific database adapters.
|
|
14
|
-
# The class uses code from: https://github.com/newrelic/newrelic-ruby-agent/blob/1fca78cc7a087421ad58088d8bea72c0362bc62f/lib/new_relic/agent/database/obfuscation_helpers.rb
|
|
15
|
-
#
|
|
16
|
-
# To use this in your instrumentation, the `Instrumentation` class for
|
|
17
|
-
# your gem must contain configuration options for:
|
|
18
|
-
# * `:db_statement`
|
|
19
|
-
# Example:
|
|
20
|
-
# `option :db_statement, default: :include, validate: %I[omit include obfuscate]`
|
|
21
|
-
# * `:obfuscation_limit`
|
|
22
|
-
# Example:
|
|
23
|
-
# `option :obfuscation_limit, default: 2000, validate: :integer`
|
|
24
|
-
#
|
|
25
|
-
# If you want to add support for a new adapter, update the following
|
|
26
|
-
# constants to include keys for your adapter:
|
|
27
|
-
# * DIALECT_COMPONENTS
|
|
28
|
-
# * CLEANUP_REGEX
|
|
29
|
-
# You must also add a new constant that uses `generate_regex` with your
|
|
30
|
-
# adapter's dialect components that is named like
|
|
31
|
-
# `<ADAPTER>_COMPONENTS_REGEX`, such as: `MYSQL_COMPONENTS_REGEX`.
|
|
32
|
-
#
|
|
33
|
-
# @api public
|
|
34
|
-
module SqlObfuscation
|
|
35
|
-
module_function
|
|
36
|
-
|
|
37
|
-
# From: https://github.com/newrelic/newrelic-ruby-agent/blob/1fca78cc7a087421ad58088d8bea72c0362bc62f/lib/new_relic/agent/database/obfuscation_helpers.rb
|
|
38
|
-
COMPONENTS_REGEX_MAP = {
|
|
39
|
-
single_quotes: /'(?:[^']|'')*?(?:\\'.*|'(?!'))/,
|
|
40
|
-
double_quotes: /"(?:[^"]|"")*?(?:\\".*|"(?!"))/,
|
|
41
|
-
dollar_quotes: /(\$(?!\d)[^$]*?\$).*?(?:\1|$)/,
|
|
42
|
-
uuids: /\{?(?:[0-9a-fA-F]\-*){32}\}?/,
|
|
43
|
-
numeric_literals: /-?\b(?:[0-9]+\.)?[0-9]+([eE][+-]?[0-9]+)?\b/,
|
|
44
|
-
boolean_literals: /\b(?:true|false|null)\b/i,
|
|
45
|
-
hexadecimal_literals: /0x[0-9a-fA-F]+/,
|
|
46
|
-
comments: /(?:#|--).*?(?=\r|\n|$)/i,
|
|
47
|
-
multi_line_comments: %r{(?:\/\*.*?\*\/)}m,
|
|
48
|
-
oracle_quoted_strings: /q'\[.*?(?:\]'|$)|q'\{.*?(?:\}'|$)|q'\<.*?(?:\>'|$)|q'\(.*?(?:\)'|$)/
|
|
49
|
-
}.freeze
|
|
50
|
-
|
|
51
|
-
DIALECT_COMPONENTS = {
|
|
52
|
-
default: COMPONENTS_REGEX_MAP.keys,
|
|
53
|
-
mysql: %i[single_quotes double_quotes numeric_literals boolean_literals
|
|
54
|
-
hexadecimal_literals comments multi_line_comments],
|
|
55
|
-
postgres: %i[single_quotes dollar_quotes uuids numeric_literals
|
|
56
|
-
boolean_literals comments multi_line_comments],
|
|
57
|
-
sqlite: %i[single_quotes numeric_literals boolean_literals hexadecimal_literals
|
|
58
|
-
comments multi_line_comments],
|
|
59
|
-
oracle: %i[single_quotes oracle_quoted_strings numeric_literals comments
|
|
60
|
-
multi_line_comments],
|
|
61
|
-
cassandra: %i[single_quotes uuids numeric_literals boolean_literals
|
|
62
|
-
hexadecimal_literals comments multi_line_comments]
|
|
63
|
-
}.freeze
|
|
64
|
-
|
|
65
|
-
PLACEHOLDER = '?'
|
|
66
|
-
|
|
67
|
-
# We use these to check whether the query contains any quote characters
|
|
68
|
-
# after obfuscation. If so, that's a good indication that the original
|
|
69
|
-
# query was malformed, and so our obfuscation can't reliably find
|
|
70
|
-
# literals. In such a case, we'll replace the entire query with a
|
|
71
|
-
# placeholder.
|
|
72
|
-
CLEANUP_REGEX = {
|
|
73
|
-
default: %r{'|"|\/\*|\*\/},
|
|
74
|
-
mysql: %r{'|"|\/\*|\*\//},
|
|
75
|
-
postgres: %r{'|\/\*|\*\/|\$(?!\?)/},
|
|
76
|
-
sqlite: %r{'|\/\*|\*\//},
|
|
77
|
-
cassandra: %r{'|\/\*|\*\//},
|
|
78
|
-
oracle: %r{'|\/\*|\*\//}
|
|
79
|
-
}.freeze
|
|
80
|
-
|
|
81
|
-
# @api private
|
|
82
|
-
def generate_regex(dialect)
|
|
83
|
-
components = DIALECT_COMPONENTS[dialect]
|
|
84
|
-
Regexp.union(components.map { |component| COMPONENTS_REGEX_MAP[component] })
|
|
85
|
-
end
|
|
86
|
-
|
|
87
|
-
DEFAULT_COMPONENTS_REGEX = generate_regex(:default)
|
|
88
|
-
MYSQL_COMPONENTS_REGEX = generate_regex(:mysql)
|
|
89
|
-
POSTGRES_COMPONENTS_REGEX = generate_regex(:postgres)
|
|
90
|
-
SQLITE_COMPONENTS_REGEX = generate_regex(:sqlite)
|
|
91
|
-
CASSANDRA_COMPONENTS_REGEX = generate_regex(:cassandra)
|
|
92
|
-
ORACLE_COMPONENTS_REGEX = generate_regex(:oracle)
|
|
93
|
-
|
|
94
|
-
# This is a SQL obfuscation utility intended for use in database adapter instrumentation.
|
|
95
|
-
#
|
|
96
|
-
# @param sql [String] The SQL to obfuscate.
|
|
97
|
-
# @param obfuscation_limit [optional Integer] the length at which the SQL string will not be obfuscated
|
|
98
|
-
# @param adapter [optional Symbol] the type of database adapter calling the method. `:default`, `:mysql` and `:postgres` are supported.
|
|
99
|
-
# @return [String] The SQL query string where the values are replaced with "?". When the sql statement exceeds the obfuscation limit
|
|
100
|
-
# the first matched pair from the SQL statement will be returned, with an appended truncation message. If truncation is unsuccessful,
|
|
101
|
-
# a string describing the error will be returned.
|
|
102
|
-
#
|
|
103
|
-
# @api public
|
|
104
|
-
def obfuscate_sql(sql, obfuscation_limit: 2000, adapter: :default)
|
|
105
|
-
return "SQL not obfuscated, query exceeds #{obfuscation_limit} characters" if sql.size > obfuscation_limit
|
|
106
|
-
|
|
107
|
-
regex = case adapter
|
|
108
|
-
when :mysql
|
|
109
|
-
MYSQL_COMPONENTS_REGEX
|
|
110
|
-
when :postgres
|
|
111
|
-
POSTGRES_COMPONENTS_REGEX
|
|
112
|
-
else
|
|
113
|
-
DEFAULT_COMPONENTS_REGEX
|
|
114
|
-
end
|
|
115
|
-
|
|
116
|
-
# Original MySQL UTF-8 Encoding Fixes:
|
|
117
|
-
# https://github.com/open-telemetry/opentelemetry-ruby-contrib/pull/160
|
|
118
|
-
# https://github.com/open-telemetry/opentelemetry-ruby-contrib/pull/345
|
|
119
|
-
sql = OpenTelemetry::Common::Utilities.utf8_encode(sql, binary: true)
|
|
120
|
-
|
|
121
|
-
sql = sql.gsub(regex, PLACEHOLDER)
|
|
122
|
-
return 'Failed to obfuscate SQL query - quote characters remained after obfuscation' if CLEANUP_REGEX[adapter].match(sql)
|
|
123
|
-
|
|
124
|
-
sql
|
|
125
|
-
rescue StandardError => e
|
|
126
|
-
OpenTelemetry.handle_error(message: 'Failed to obfuscate SQL', exception: e)
|
|
127
|
-
end
|
|
128
|
-
end
|
|
129
|
-
end
|
|
130
|
-
end
|