sentrace 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE.txt +21 -0
- data/README.md +192 -0
- data/app/controllers/sentrace/application_controller.rb +4 -0
- data/app/controllers/sentrace/ingestion_controller.rb +169 -0
- data/app/controllers/sentrace/internal/base_controller.rb +47 -0
- data/app/controllers/sentrace/internal/events_controller.rb +25 -0
- data/app/controllers/sentrace/internal/issues_controller.rb +36 -0
- data/app/controllers/sentrace/internal/projects_controller.rb +58 -0
- data/app/controllers/sentrace/internal/stats_controller.rb +10 -0
- data/app/controllers/sentrace/mcp_controller.rb +74 -0
- data/app/jobs/sentrace/retention_cleanup_job.rb +9 -0
- data/app/models/sentrace/application_record.rb +5 -0
- data/app/models/sentrace/event.rb +41 -0
- data/app/models/sentrace/issue.rb +32 -0
- data/app/models/sentrace/mcp_token.rb +82 -0
- data/app/models/sentrace/project.rb +50 -0
- data/app/models/sentrace/project_key.rb +60 -0
- data/config/routes.rb +11 -0
- data/db/migrate/20261116000001_create_sentrace_projects.rb +12 -0
- data/db/migrate/20261116000002_create_sentrace_project_keys.rb +14 -0
- data/db/migrate/20261116000003_create_sentrace_issues.rb +25 -0
- data/db/migrate/20261116000004_create_sentrace_events.rb +31 -0
- data/db/migrate/20261116000005_create_sentrace_mcp_tokens.rb +17 -0
- data/lib/generators/sentrace/install/install_generator.rb +91 -0
- data/lib/generators/sentrace/install/templates/initializer.rb.tt +34 -0
- data/lib/generators/sentrace/install/templates/sentrace_setup.rake.tt +21 -0
- data/lib/sentrace/benchmark/storage_runner.rb +289 -0
- data/lib/sentrace/configuration.rb +198 -0
- data/lib/sentrace/dsn.rb +123 -0
- data/lib/sentrace/engine.rb +10 -0
- data/lib/sentrace/errors.rb +20 -0
- data/lib/sentrace/ingestion/envelope_parser.rb +143 -0
- data/lib/sentrace/ingestion/event_normalizer.rb +315 -0
- data/lib/sentrace/ingestion/processor.rb +90 -0
- data/lib/sentrace/ingestion/sanitizer.rb +293 -0
- data/lib/sentrace/issues/fingerprint.rb +177 -0
- data/lib/sentrace/issues/grouper.rb +184 -0
- data/lib/sentrace/mcp/dispatcher.rb +23 -0
- data/lib/sentrace/mcp/environment_router.rb +266 -0
- data/lib/sentrace/mcp/local_backend.rb +425 -0
- data/lib/sentrace/mcp/remote_http_client.rb +310 -0
- data/lib/sentrace/mcp/server.rb +417 -0
- data/lib/sentrace/mcp/stdio_runner.rb +55 -0
- data/lib/sentrace/mcp/token_manager.rb +73 -0
- data/lib/sentrace/project_manager.rb +64 -0
- data/lib/sentrace/retention/cleaner.rb +105 -0
- data/lib/sentrace/storage_stats.rb +136 -0
- data/lib/sentrace/version.rb +3 -0
- data/lib/sentrace.rb +40 -0
- data/lib/tasks/sentrace_benchmark.rake +9 -0
- data/lib/tasks/sentrace_mcp.rake +36 -0
- data/lib/tasks/sentrace_projects.rake +22 -0
- data/lib/tasks/sentrace_retention.rake +16 -0
- metadata +148 -0
|
@@ -0,0 +1,143 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
|
|
3
|
+
module Sentrace
|
|
4
|
+
module Ingestion
|
|
5
|
+
class EnvelopeParser
|
|
6
|
+
ParsedEnvelope = Data.define(:headers, :events, :discarded_item_types)
|
|
7
|
+
|
|
8
|
+
MAX_HEADER_LINE_BYTES = 16 * 1024
|
|
9
|
+
|
|
10
|
+
class << self
|
|
11
|
+
def parse(raw_body)
|
|
12
|
+
new.parse(raw_body)
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def extract_header_dsn(raw_body)
|
|
16
|
+
return nil if raw_body.nil? || raw_body.empty?
|
|
17
|
+
|
|
18
|
+
binary = raw_body.b
|
|
19
|
+
newline_idx = binary.index("\n")
|
|
20
|
+
first_line = newline_idx ? binary.byteslice(0, newline_idx) : binary
|
|
21
|
+
return nil if first_line.bytesize > MAX_HEADER_LINE_BYTES
|
|
22
|
+
|
|
23
|
+
parsed = JSON.parse(first_line.force_encoding(Encoding::UTF_8))
|
|
24
|
+
parsed.is_a?(Hash) ? parsed["dsn"] : nil
|
|
25
|
+
rescue JSON::ParserError, EncodingError
|
|
26
|
+
nil
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def parse(raw_body)
|
|
31
|
+
if raw_body.nil? || raw_body.bytesize.zero?
|
|
32
|
+
raise EnvelopeParseError, "Envelope body cannot be empty"
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
buffer = raw_body.b
|
|
36
|
+
cursor = 0
|
|
37
|
+
total = buffer.bytesize
|
|
38
|
+
|
|
39
|
+
header_line, cursor = read_line(buffer, cursor, total)
|
|
40
|
+
if header_line.nil? || header_line.empty?
|
|
41
|
+
raise EnvelopeParseError, "Missing envelope header"
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
envelope_headers = parse_json_object(header_line, "envelope header")
|
|
45
|
+
events = []
|
|
46
|
+
discarded = []
|
|
47
|
+
|
|
48
|
+
while cursor < total
|
|
49
|
+
# Skip trailing newline(s) at the end of an envelope
|
|
50
|
+
if buffer.getbyte(cursor) == 10
|
|
51
|
+
cursor += 1
|
|
52
|
+
next
|
|
53
|
+
elsif buffer.getbyte(cursor) == 13 && cursor + 1 < total && buffer.getbyte(cursor + 1) == 10
|
|
54
|
+
cursor += 2
|
|
55
|
+
next
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
item_header_line, cursor = read_line(buffer, cursor, total)
|
|
59
|
+
break if item_header_line.nil? || item_header_line.empty?
|
|
60
|
+
|
|
61
|
+
item_header = parse_json_object(item_header_line, "envelope item header")
|
|
62
|
+
item_type = (item_header["type"] || "event").to_s
|
|
63
|
+
|
|
64
|
+
payload_bytes, cursor = read_item_payload(buffer, cursor, total, item_header)
|
|
65
|
+
|
|
66
|
+
if item_type == "event"
|
|
67
|
+
event_hash = parse_json_object(payload_bytes, "event payload")
|
|
68
|
+
if event_hash["type"].to_s == "transaction"
|
|
69
|
+
discarded << "transaction"
|
|
70
|
+
else
|
|
71
|
+
events << event_hash
|
|
72
|
+
end
|
|
73
|
+
else
|
|
74
|
+
discarded << item_type
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
ParsedEnvelope.new(
|
|
79
|
+
headers: envelope_headers,
|
|
80
|
+
events: events,
|
|
81
|
+
discarded_item_types: discarded
|
|
82
|
+
)
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
private
|
|
86
|
+
def read_line(buffer, cursor, total)
|
|
87
|
+
return [ nil, total ] if cursor >= total
|
|
88
|
+
|
|
89
|
+
newline_idx = buffer.index("\n", cursor)
|
|
90
|
+
if newline_idx
|
|
91
|
+
line = buffer.byteslice(cursor, newline_idx - cursor)
|
|
92
|
+
line = line.byteslice(0, line.bytesize - 1) if line.end_with?("\r")
|
|
93
|
+
[ line, newline_idx + 1 ]
|
|
94
|
+
else
|
|
95
|
+
line = buffer.byteslice(cursor, total - cursor)
|
|
96
|
+
line = line.byteslice(0, line.bytesize - 1) if line.end_with?("\r")
|
|
97
|
+
[ line, total ]
|
|
98
|
+
end
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def read_item_payload(buffer, cursor, total, item_header)
|
|
102
|
+
if item_header.key?("length")
|
|
103
|
+
length = item_header["length"]
|
|
104
|
+
unless length.is_a?(Integer) && length >= 0
|
|
105
|
+
raise EnvelopeParseError, "Invalid item length in envelope item header"
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
if cursor + length > total
|
|
109
|
+
raise EnvelopeParseError, "Truncated envelope item payload (expected #{length} bytes)"
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
payload = buffer.byteslice(cursor, length)
|
|
113
|
+
next_cursor = cursor + length
|
|
114
|
+
if next_cursor < total && buffer.getbyte(next_cursor) == 13 && next_cursor + 1 < total && buffer.getbyte(next_cursor + 1) == 10
|
|
115
|
+
next_cursor += 2
|
|
116
|
+
elsif next_cursor < total && buffer.getbyte(next_cursor) == 10
|
|
117
|
+
next_cursor += 1
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
[ payload, next_cursor ]
|
|
121
|
+
else
|
|
122
|
+
read_line(buffer, cursor, total)
|
|
123
|
+
end
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def parse_json_object(raw_bytes, label)
|
|
127
|
+
utf8 = raw_bytes.dup.force_encoding(Encoding::UTF_8)
|
|
128
|
+
unless utf8.valid_encoding?
|
|
129
|
+
raise EnvelopeParseError, "Invalid UTF-8 encoding in #{label}"
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
parsed = JSON.parse(utf8)
|
|
133
|
+
unless parsed.is_a?(Hash)
|
|
134
|
+
raise EnvelopeParseError, "Expected JSON object for #{label}"
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
parsed
|
|
138
|
+
rescue JSON::ParserError
|
|
139
|
+
raise EnvelopeParseError, "Invalid JSON in #{label}"
|
|
140
|
+
end
|
|
141
|
+
end
|
|
142
|
+
end
|
|
143
|
+
end
|
|
@@ -0,0 +1,315 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
require "time"
|
|
3
|
+
|
|
4
|
+
module Sentrace
|
|
5
|
+
module Ingestion
|
|
6
|
+
class EventNormalizer
|
|
7
|
+
NormalizedEvent = Data.define(
|
|
8
|
+
:event_id,
|
|
9
|
+
:occurred_at,
|
|
10
|
+
:received_at,
|
|
11
|
+
:environment,
|
|
12
|
+
:release,
|
|
13
|
+
:platform,
|
|
14
|
+
:level,
|
|
15
|
+
:logger,
|
|
16
|
+
:culprit,
|
|
17
|
+
:exception_type,
|
|
18
|
+
:issue_message,
|
|
19
|
+
:fingerprint,
|
|
20
|
+
:event_data,
|
|
21
|
+
:raw_user,
|
|
22
|
+
:raw_request
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
EVENT_ID_HEX = /\A[0-9a-f]{32}\z/
|
|
26
|
+
ISSUE_MESSAGE_MAX_CHARS = 1024
|
|
27
|
+
|
|
28
|
+
LEVEL_MAP = {
|
|
29
|
+
"debug" => :debug,
|
|
30
|
+
"info" => :info,
|
|
31
|
+
"warn" => :warning,
|
|
32
|
+
"warning" => :warning,
|
|
33
|
+
"error" => :error,
|
|
34
|
+
"fatal" => :fatal
|
|
35
|
+
}.freeze
|
|
36
|
+
|
|
37
|
+
class << self
|
|
38
|
+
def normalize(raw_event, envelope_headers: {}, received_at: Time.current, config: Sentrace.configuration)
|
|
39
|
+
new(config: config).normalize(raw_event, envelope_headers: envelope_headers, received_at: received_at)
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def truncate_utf8_bytes(value, max_bytes)
|
|
43
|
+
return nil if value.nil?
|
|
44
|
+
|
|
45
|
+
str = value.to_s.dup.force_encoding(Encoding::UTF_8).scrub
|
|
46
|
+
return str if str.bytesize <= max_bytes
|
|
47
|
+
|
|
48
|
+
sliced = str.byteslice(0, max_bytes)
|
|
49
|
+
until sliced.valid_encoding? || sliced.empty?
|
|
50
|
+
sliced = sliced.byteslice(0, sliced.bytesize - 1)
|
|
51
|
+
end
|
|
52
|
+
sliced
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def initialize(config: Sentrace.configuration)
|
|
57
|
+
@config = config
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def normalize(raw_event, envelope_headers: {}, received_at: Time.current)
|
|
61
|
+
unless raw_event.is_a?(Hash)
|
|
62
|
+
raise EnvelopeParseError, "Event payload must be a JSON object"
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
event_id = extract_event_id(raw_event, envelope_headers)
|
|
66
|
+
occurred_at = extract_timestamp(raw_event["timestamp"], fallback: received_at)
|
|
67
|
+
level = normalize_level(raw_event["level"])
|
|
68
|
+
|
|
69
|
+
exception_data = extract_exception(raw_event["exception"])
|
|
70
|
+
message = extract_message(raw_event)
|
|
71
|
+
culprit = truncate_chars(raw_event["culprit"] || raw_event["transaction"] || derive_culprit(exception_data), 512)
|
|
72
|
+
|
|
73
|
+
summary_source = exception_data ? exception_data["value"] : message
|
|
74
|
+
issue_message = truncate_chars(summary_source, ISSUE_MESSAGE_MAX_CHARS)
|
|
75
|
+
|
|
76
|
+
event_data = {
|
|
77
|
+
"exception" => exception_data,
|
|
78
|
+
"message" => message,
|
|
79
|
+
"request" => nil,
|
|
80
|
+
"breadcrumbs" => extract_breadcrumbs(raw_event["breadcrumbs"]),
|
|
81
|
+
"tags" => extract_tags(raw_event["tags"]),
|
|
82
|
+
"contexts" => extract_contexts(raw_event["contexts"]),
|
|
83
|
+
"user" => nil
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
NormalizedEvent.new(
|
|
87
|
+
event_id: event_id,
|
|
88
|
+
occurred_at: occurred_at,
|
|
89
|
+
received_at: received_at,
|
|
90
|
+
environment: truncate_chars(raw_event["environment"], 64),
|
|
91
|
+
release: truncate_chars(raw_event["release"], 128),
|
|
92
|
+
platform: truncate_chars(raw_event["platform"], 32),
|
|
93
|
+
level: level,
|
|
94
|
+
logger: truncate_chars(raw_event["logger"], 128),
|
|
95
|
+
culprit: culprit,
|
|
96
|
+
exception_type: exception_data ? exception_data["type"] : nil,
|
|
97
|
+
issue_message: issue_message,
|
|
98
|
+
fingerprint: extract_fingerprint(raw_event["fingerprint"]),
|
|
99
|
+
event_data: event_data,
|
|
100
|
+
raw_user: raw_event["user"].is_a?(Hash) ? raw_event["user"] : nil,
|
|
101
|
+
raw_request: raw_event["request"].is_a?(Hash) ? raw_event["request"] : nil
|
|
102
|
+
)
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
private
|
|
106
|
+
def extract_event_id(raw_event, envelope_headers)
|
|
107
|
+
candidate = raw_event["event_id"] || envelope_headers["event_id"]
|
|
108
|
+
normalized = candidate.to_s.strip.delete("-").downcase
|
|
109
|
+
unless normalized.match?(EVENT_ID_HEX)
|
|
110
|
+
raise EnvelopeParseError, "Missing or invalid Sentry event_id"
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
normalized
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def extract_timestamp(raw_timestamp, fallback:)
|
|
117
|
+
return fallback if raw_timestamp.nil? || raw_timestamp.to_s.strip.empty?
|
|
118
|
+
|
|
119
|
+
case raw_timestamp
|
|
120
|
+
when Numeric
|
|
121
|
+
Time.at(raw_timestamp).utc
|
|
122
|
+
when String
|
|
123
|
+
if raw_timestamp.match?(/\A\d+(?:\.\d+)?\z/)
|
|
124
|
+
Time.at(raw_timestamp.to_f).utc
|
|
125
|
+
else
|
|
126
|
+
Time.iso8601(raw_timestamp).utc
|
|
127
|
+
end
|
|
128
|
+
else
|
|
129
|
+
raise EnvelopeParseError, "Invalid event timestamp"
|
|
130
|
+
end
|
|
131
|
+
rescue ArgumentError, RangeError
|
|
132
|
+
raise EnvelopeParseError, "Malformed event timestamp"
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
def normalize_level(raw_level)
|
|
136
|
+
return :error if raw_level.nil?
|
|
137
|
+
|
|
138
|
+
LEVEL_MAP.fetch(raw_level.to_s.strip.downcase, :error)
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
def extract_message(raw_event)
|
|
142
|
+
raw_msg =
|
|
143
|
+
if raw_event["message"].is_a?(String)
|
|
144
|
+
raw_event["message"]
|
|
145
|
+
elsif raw_event["message"].is_a?(Hash)
|
|
146
|
+
raw_event.dig("message", "formatted") || raw_event.dig("message", "message")
|
|
147
|
+
elsif raw_event["logentry"].is_a?(Hash)
|
|
148
|
+
raw_event.dig("logentry", "formatted") || raw_event.dig("logentry", "message")
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
return nil if raw_msg.nil? || raw_msg.empty?
|
|
152
|
+
|
|
153
|
+
self.class.truncate_utf8_bytes(raw_msg, @config.max_message_bytes)
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
def extract_exception(raw_exception)
|
|
157
|
+
values =
|
|
158
|
+
if raw_exception.is_a?(Hash) && raw_exception["values"].is_a?(Array)
|
|
159
|
+
raw_exception["values"]
|
|
160
|
+
elsif raw_exception.is_a?(Array)
|
|
161
|
+
raw_exception
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
return nil if values.nil? || values.empty?
|
|
165
|
+
|
|
166
|
+
primary = values.reverse.find { |item| item.is_a?(Hash) && (item["type"] || item["value"]) }
|
|
167
|
+
return nil unless primary
|
|
168
|
+
|
|
169
|
+
raw_frames =
|
|
170
|
+
if primary.dig("stacktrace", "frames").is_a?(Array)
|
|
171
|
+
primary.dig("stacktrace", "frames")
|
|
172
|
+
else
|
|
173
|
+
# Fall back to earliest cause carrying a stacktrace when wrapper reused backtrace
|
|
174
|
+
with_trace = values.reverse.find { |item| item.is_a?(Hash) && item.dig("stacktrace", "frames").is_a?(Array) }
|
|
175
|
+
with_trace ? with_trace.dig("stacktrace", "frames") : []
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
bounded_frames = raw_frames.select { |f| f.is_a?(Hash) }.last(@config.max_stack_frames).map do |frame|
|
|
179
|
+
{
|
|
180
|
+
"filename" => truncate_chars(frame["filename"] || frame["abs_path"], 512),
|
|
181
|
+
"function" => truncate_chars(frame["function"], 255),
|
|
182
|
+
"lineno" => frame["lineno"].is_a?(Integer) ? frame["lineno"] : nil,
|
|
183
|
+
"in_app" => frame["in_app"] == true
|
|
184
|
+
}.compact
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
{
|
|
188
|
+
"type" => truncate_chars(primary["type"], 255),
|
|
189
|
+
"value" => self.class.truncate_utf8_bytes(primary["value"], @config.max_message_bytes),
|
|
190
|
+
"frames" => bounded_frames
|
|
191
|
+
}
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
def derive_culprit(exception_data)
|
|
195
|
+
return nil unless exception_data && exception_data["frames"].is_a?(Array)
|
|
196
|
+
|
|
197
|
+
in_app_frame = exception_data["frames"].reverse.find { |f| f["in_app"] } || exception_data["frames"].last
|
|
198
|
+
return nil unless in_app_frame
|
|
199
|
+
|
|
200
|
+
parts = [ in_app_frame["filename"], in_app_frame["function"] ].compact
|
|
201
|
+
parts.empty? ? nil : parts.join(" in ")
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
def extract_breadcrumbs(raw_breadcrumbs)
|
|
205
|
+
items =
|
|
206
|
+
if raw_breadcrumbs.is_a?(Hash) && raw_breadcrumbs["values"].is_a?(Array)
|
|
207
|
+
raw_breadcrumbs["values"]
|
|
208
|
+
elsif raw_breadcrumbs.is_a?(Array)
|
|
209
|
+
raw_breadcrumbs
|
|
210
|
+
else
|
|
211
|
+
[]
|
|
212
|
+
end
|
|
213
|
+
|
|
214
|
+
items.select { |b| b.is_a?(Hash) }.last(@config.max_breadcrumbs).map do |crumb|
|
|
215
|
+
entry = {
|
|
216
|
+
"timestamp" => crumb["timestamp"]&.to_s,
|
|
217
|
+
"type" => truncate_chars(crumb["type"], 64),
|
|
218
|
+
"category" => truncate_chars(crumb["category"], 128),
|
|
219
|
+
"level" => truncate_chars(crumb["level"], 32),
|
|
220
|
+
"message" => self.class.truncate_utf8_bytes(crumb["message"], 1024)
|
|
221
|
+
}.compact
|
|
222
|
+
|
|
223
|
+
if crumb["data"].is_a?(Hash) && crumb["data"].any?
|
|
224
|
+
entry["data"] = crumb["data"].first(16).to_h do |k, v|
|
|
225
|
+
[
|
|
226
|
+
self.class.truncate_utf8_bytes(k.to_s, 64),
|
|
227
|
+
v.is_a?(Numeric) || v == true || v == false || v.nil? ? v : self.class.truncate_utf8_bytes(v.to_s, 256)
|
|
228
|
+
]
|
|
229
|
+
end
|
|
230
|
+
end
|
|
231
|
+
|
|
232
|
+
entry
|
|
233
|
+
end
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
def extract_tags(raw_tags)
|
|
237
|
+
pairs =
|
|
238
|
+
case raw_tags
|
|
239
|
+
when Hash
|
|
240
|
+
raw_tags.to_a
|
|
241
|
+
when Array
|
|
242
|
+
raw_tags.select { |item| item.is_a?(Array) && item.size == 2 }
|
|
243
|
+
else
|
|
244
|
+
[]
|
|
245
|
+
end
|
|
246
|
+
|
|
247
|
+
pairs.first(@config.max_tags).each_with_object({}) do |(raw_key, raw_val), acc|
|
|
248
|
+
next if raw_key.nil?
|
|
249
|
+
|
|
250
|
+
key = self.class.truncate_utf8_bytes(raw_key.to_s.strip, @config.max_tag_key_bytes)
|
|
251
|
+
next if key.nil? || key.empty?
|
|
252
|
+
|
|
253
|
+
acc[key] = raw_val.nil? ? nil : self.class.truncate_utf8_bytes(raw_val.to_s, @config.max_tag_value_bytes)
|
|
254
|
+
end
|
|
255
|
+
end
|
|
256
|
+
|
|
257
|
+
def extract_contexts(raw_contexts)
|
|
258
|
+
return {} unless raw_contexts.is_a?(Hash)
|
|
259
|
+
|
|
260
|
+
bounded = {}
|
|
261
|
+
raw_contexts.each do |ctx_key, ctx_val|
|
|
262
|
+
break if bounded.size >= @config.max_context_entries
|
|
263
|
+
next unless ctx_val.is_a?(Hash)
|
|
264
|
+
|
|
265
|
+
key_str = self.class.truncate_utf8_bytes(ctx_key.to_s, 64)
|
|
266
|
+
next if key_str.nil? || key_str.empty?
|
|
267
|
+
|
|
268
|
+
candidate = bounded.merge(key_str => bound_nested_value(ctx_val, depth: 0))
|
|
269
|
+
if JSON.generate(candidate).bytesize <= @config.max_context_bytes
|
|
270
|
+
bounded = candidate
|
|
271
|
+
end
|
|
272
|
+
end
|
|
273
|
+
|
|
274
|
+
bounded
|
|
275
|
+
end
|
|
276
|
+
|
|
277
|
+
def bound_nested_value(value, depth:)
|
|
278
|
+
return "[TRUNCATED]" if depth > 3
|
|
279
|
+
|
|
280
|
+
case value
|
|
281
|
+
when Hash
|
|
282
|
+
value.first(16).to_h do |k, v|
|
|
283
|
+
[ self.class.truncate_utf8_bytes(k.to_s, 64), bound_nested_value(v, depth: depth + 1) ]
|
|
284
|
+
end
|
|
285
|
+
when Array
|
|
286
|
+
value.first(16).map { |v| bound_nested_value(v, depth: depth + 1) }
|
|
287
|
+
when Numeric, TrueClass, FalseClass, NilClass
|
|
288
|
+
value
|
|
289
|
+
else
|
|
290
|
+
self.class.truncate_utf8_bytes(value.to_s, 512)
|
|
291
|
+
end
|
|
292
|
+
end
|
|
293
|
+
|
|
294
|
+
def extract_fingerprint(raw_fingerprint)
|
|
295
|
+
return nil unless raw_fingerprint.is_a?(Array)
|
|
296
|
+
|
|
297
|
+
cleaned = raw_fingerprint.filter_map do |part|
|
|
298
|
+
str = part.to_s.strip
|
|
299
|
+
str unless str.empty?
|
|
300
|
+
end
|
|
301
|
+
|
|
302
|
+
cleaned.empty? ? nil : cleaned
|
|
303
|
+
end
|
|
304
|
+
|
|
305
|
+
def truncate_chars(value, max_chars)
|
|
306
|
+
return nil if value.nil?
|
|
307
|
+
|
|
308
|
+
str = value.to_s.dup.force_encoding(Encoding::UTF_8).scrub.strip
|
|
309
|
+
return nil if str.empty?
|
|
310
|
+
|
|
311
|
+
str.length > max_chars ? str[0, max_chars] : str
|
|
312
|
+
end
|
|
313
|
+
end
|
|
314
|
+
end
|
|
315
|
+
end
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
module Sentrace
|
|
2
|
+
module Ingestion
|
|
3
|
+
class Processor
|
|
4
|
+
EnvelopeResult = Data.define(:primary_event_id, :accepted_event_ids, :duplicate_event_ids, :discarded_count)
|
|
5
|
+
|
|
6
|
+
class << self
|
|
7
|
+
def process_raw_envelope(raw_body, project:, received_at: Time.current, config: Sentrace.configuration)
|
|
8
|
+
new(config: config).process_raw_envelope(raw_body, project: project, received_at: received_at)
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def ingest_event(raw_event, project:, envelope_headers: {}, received_at: Time.current, config: Sentrace.configuration)
|
|
12
|
+
new(config: config).ingest_event(
|
|
13
|
+
raw_event,
|
|
14
|
+
project: project,
|
|
15
|
+
envelope_headers: envelope_headers,
|
|
16
|
+
received_at: received_at
|
|
17
|
+
)
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def initialize(config: Sentrace.configuration)
|
|
22
|
+
@config = config
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def process_raw_envelope(raw_body, project:, received_at: Time.current)
|
|
26
|
+
parsed = EnvelopeParser.parse(raw_body)
|
|
27
|
+
accepted_ids = []
|
|
28
|
+
duplicate_ids = []
|
|
29
|
+
|
|
30
|
+
parsed.events.each do |raw_event|
|
|
31
|
+
result = ingest_event(
|
|
32
|
+
raw_event,
|
|
33
|
+
project: project,
|
|
34
|
+
envelope_headers: parsed.headers,
|
|
35
|
+
received_at: received_at
|
|
36
|
+
)
|
|
37
|
+
if result.duplicate
|
|
38
|
+
duplicate_ids << result.event_id
|
|
39
|
+
else
|
|
40
|
+
accepted_ids << result.event_id
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
primary_id = accepted_ids.first || duplicate_ids.first || extract_valid_header_event_id(parsed.headers)
|
|
45
|
+
|
|
46
|
+
EnvelopeResult.new(
|
|
47
|
+
primary_event_id: primary_id,
|
|
48
|
+
accepted_event_ids: accepted_ids,
|
|
49
|
+
duplicate_event_ids: duplicate_ids,
|
|
50
|
+
discarded_count: parsed.discarded_item_types.size
|
|
51
|
+
)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
def ingest_event(raw_event, project:, envelope_headers: {}, received_at: Time.current)
|
|
55
|
+
project_id = project.is_a?(Integer) ? project : project.id
|
|
56
|
+
normalized = EventNormalizer.normalize(
|
|
57
|
+
raw_event,
|
|
58
|
+
envelope_headers: envelope_headers,
|
|
59
|
+
received_at: received_at,
|
|
60
|
+
config: @config
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
# Fast-path read check via unique index (project_id, event_id) before running sanitizer or writes
|
|
64
|
+
if Event.exists?(project_id: project_id, event_id: normalized.event_id)
|
|
65
|
+
return Issues::Grouper::PersistResult.new(
|
|
66
|
+
event_id: normalized.event_id,
|
|
67
|
+
event_record_id: nil,
|
|
68
|
+
issue_id: nil,
|
|
69
|
+
duplicate: true
|
|
70
|
+
)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
sanitized = Sanitizer.sanitize(normalized, project_id: project_id, config: @config)
|
|
74
|
+
fingerprint = Issues::Fingerprint.compute(sanitized)
|
|
75
|
+
|
|
76
|
+
Issues::Grouper.persist!(
|
|
77
|
+
project_id: project_id,
|
|
78
|
+
normalized_event: sanitized,
|
|
79
|
+
fingerprint: fingerprint
|
|
80
|
+
)
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
private
|
|
84
|
+
def extract_valid_header_event_id(headers)
|
|
85
|
+
candidate = headers["event_id"].to_s.strip.delete("-").downcase
|
|
86
|
+
candidate.match?(EventNormalizer::EVENT_ID_HEX) ? candidate : nil
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
end
|