lesli_audit 1.1.2 → 1.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/assets/stylesheets/lesli_audit/application.css +183 -0
- data/app/controllers/lesli_audit/analytics_controller.rb +16 -0
- data/app/controllers/lesli_audit/logs_controller.rb +12 -2
- data/app/controllers/lesli_audit/traffic_controller.rb +25 -0
- data/app/controllers/lesli_audit/users_controller.rb +7 -4
- data/app/interfaces/lesli_audit/logger_interface.rb +5 -4
- data/app/jobs/lesli_audit/cloudflare_traffic_sync_job.rb +78 -0
- data/app/models/lesli_audit/account.rb +12 -0
- data/app/models/lesli_audit/cloudflare_traffic_sync.rb +99 -0
- data/app/models/lesli_audit/traffic_breakdown.rb +38 -0
- data/app/models/lesli_audit/traffic_rollup.rb +33 -0
- data/app/models/lesli_audit/user_journal.rb +0 -2
- data/app/services/lesli_audit/activity_audit.rb +8 -4
- data/app/services/lesli_audit/analytics_service.rb +237 -0
- data/app/services/lesli_audit/cloudflare/configuration.rb +92 -0
- data/app/services/lesli_audit/cloudflare/graphql_client.rb +309 -0
- data/app/services/lesli_audit/cloudflare/snapshot_converter.rb +101 -0
- data/app/services/lesli_audit/cloudflare/sync_service.rb +79 -0
- data/app/services/lesli_audit/cloudflare_traffic_snapshot.rb +182 -0
- data/app/services/lesli_audit/log_service.rb +99 -5
- data/app/services/lesli_audit/request_service.rb +73 -10
- data/app/services/lesli_audit/retention_cleanup.rb +16 -16
- data/app/services/lesli_audit/traffic_service.rb +216 -0
- data/app/services/lesli_audit/traffic_snapshot_importer.rb +120 -0
- data/app/services/lesli_audit/user_service.rb +119 -152
- data/app/views/lesli_audit/{visitors → analytics}/index.html.erb +40 -30
- data/app/views/lesli_audit/logs/index.html.erb +88 -19
- data/app/views/lesli_audit/logs/show.html.erb +36 -8
- data/app/views/lesli_audit/partials/_navigation.html.erb +5 -4
- data/app/views/lesli_audit/requests/index.html.erb +80 -19
- data/app/views/lesli_audit/traffic/_breakdown.html.erb +46 -0
- data/app/views/lesli_audit/traffic/index.html.erb +177 -0
- data/app/views/lesli_audit/users/index.html.erb +97 -40
- data/config/locales/translations.en.yml +137 -1
- data/config/locales/translations.es.yml +137 -1
- data/config/locales/translations.fr.yml +132 -1
- data/config/locales/translations.it.yml +132 -1
- data/config/locales/translations.pt.yml +132 -1
- data/config/routes.rb +6 -23
- data/db/migrate/v1.0/0501110110_create_lesli_audit_account_logs.rb +2 -1
- data/db/migrate/v1.0/0501120110_create_lesli_audit_user_logs.rb +2 -1
- data/db/migrate/v1.0/0501120210_create_lesli_audit_user_journals.rb +0 -3
- data/db/migrate/v1.0/0501130110_create_lesli_audit_traffic_rollups.rb +55 -0
- data/db/migrate/v1.0/0501130210_create_lesli_audit_traffic_breakdowns.rb +41 -0
- data/db/migrate/v1.0/0501130310_create_lesli_audit_cloudflare_traffic_syncs.rb +42 -0
- data/db/seed/devices.rb +2 -1
- data/db/seed/traffic.rb +128 -0
- data/db/seed/users.rb +68 -27
- data/db/seeds.rb +3 -2
- data/lib/lesli_audit/version.rb +2 -2
- data/lib/tasks/lesli_audit_cloudflare_traffic.rake +16 -0
- data/readme.md +10 -0
- metadata +24 -5
- data/app/controllers/lesli_audit/visitors_controller.rb +0 -14
- data/app/services/lesli_audit/visitor_service.rb +0 -212
|
@@ -0,0 +1,309 @@
|
|
|
1
|
+
require "json"
|
|
2
|
+
require "net/http"
|
|
3
|
+
|
|
4
|
+
module LesliAudit::Cloudflare
|
|
5
|
+
class GraphqlClient
|
|
6
|
+
ENDPOINT = URI("https://api.cloudflare.com/client/v4/graphql")
|
|
7
|
+
TRANSIENT_HTTP_STATUSES = [ 429, 500, 502, 503, 504 ].freeze
|
|
8
|
+
TRANSIENT_GRAPHQL_ERROR = /rate.?limit|temporar|timeout|try again|unavailable|internal server/i
|
|
9
|
+
TRANSIENT_GRAPHQL_CODES = %w[
|
|
10
|
+
INTERNAL_SERVER_ERROR
|
|
11
|
+
RATE_LIMITED
|
|
12
|
+
SERVICE_UNAVAILABLE
|
|
13
|
+
TIMEOUT
|
|
14
|
+
].freeze
|
|
15
|
+
NETWORK_ERRORS = [
|
|
16
|
+
Net::OpenTimeout,
|
|
17
|
+
Net::ReadTimeout,
|
|
18
|
+
EOFError,
|
|
19
|
+
Errno::ECONNRESET,
|
|
20
|
+
Errno::ETIMEDOUT
|
|
21
|
+
].freeze
|
|
22
|
+
|
|
23
|
+
DIMENSIONS = {
|
|
24
|
+
country: "clientCountryName",
|
|
25
|
+
device: "clientDeviceType",
|
|
26
|
+
referrer: "clientRefererHost"
|
|
27
|
+
}.freeze
|
|
28
|
+
|
|
29
|
+
TrafficData = Data.define(:totals, :breakdowns)
|
|
30
|
+
|
|
31
|
+
class Error < StandardError; end
|
|
32
|
+
|
|
33
|
+
class TransientError < Error
|
|
34
|
+
attr_reader :retry_after
|
|
35
|
+
|
|
36
|
+
def initialize(message, retry_after: nil)
|
|
37
|
+
@retry_after = retry_after
|
|
38
|
+
super(message)
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
class HttpTransport
|
|
43
|
+
OPEN_TIMEOUT = 10
|
|
44
|
+
READ_TIMEOUT = 30
|
|
45
|
+
|
|
46
|
+
def call(uri:, headers:, body:)
|
|
47
|
+
request = Net::HTTP::Post.new(uri)
|
|
48
|
+
headers.each { |name, value| request[name] = value }
|
|
49
|
+
request.body = body
|
|
50
|
+
|
|
51
|
+
Net::HTTP.start(
|
|
52
|
+
uri.host,
|
|
53
|
+
uri.port,
|
|
54
|
+
use_ssl: true,
|
|
55
|
+
open_timeout: OPEN_TIMEOUT,
|
|
56
|
+
read_timeout: READ_TIMEOUT
|
|
57
|
+
) do |http|
|
|
58
|
+
http.request(request)
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def initialize(token:, page_size: 1_000, max_pages: 100, max_retries: 3,
|
|
64
|
+
transport: HttpTransport.new, sleeper: ->(seconds) { sleep(seconds) })
|
|
65
|
+
@token = token.to_s.strip
|
|
66
|
+
@configured_page_size = Integer(page_size)
|
|
67
|
+
@max_pages = Integer(max_pages)
|
|
68
|
+
@max_retries = Integer(max_retries)
|
|
69
|
+
@transport = transport
|
|
70
|
+
@sleeper = sleeper
|
|
71
|
+
|
|
72
|
+
validate_configuration!
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def traffic(zone_id:, hostname:, date:)
|
|
76
|
+
page_size = page_size_for(zone_id)
|
|
77
|
+
totals = fetch_groups(zone_id:, hostname:, date:, page_size:)
|
|
78
|
+
breakdowns = supported_dimensions(zone_id).flat_map do |dimension, field|
|
|
79
|
+
fetch_groups(zone_id:, hostname:, date:, page_size:, dimension:, field:)
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
TrafficData.new(totals:, breakdowns:)
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
private
|
|
86
|
+
|
|
87
|
+
attr_reader :configured_page_size, :max_pages, :max_retries,
|
|
88
|
+
:sleeper, :token, :transport
|
|
89
|
+
|
|
90
|
+
def validate_configuration!
|
|
91
|
+
raise ArgumentError, "token must be present" if token.empty?
|
|
92
|
+
unless configured_page_size.between?(1, 10_000)
|
|
93
|
+
raise ArgumentError, "page_size must be between 1 and 10,000"
|
|
94
|
+
end
|
|
95
|
+
raise ArgumentError, "max_pages must be positive" unless max_pages.positive?
|
|
96
|
+
raise ArgumentError, "max_retries cannot be negative" if max_retries.negative?
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def page_size_for(zone_id)
|
|
100
|
+
maximum = dataset_settings(zone_id)["maxPageSize"].presence || configured_page_size
|
|
101
|
+
[ configured_page_size, Integer(maximum) ].min
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def supported_dimensions(zone_id)
|
|
105
|
+
available_fields = Array(dataset_settings(zone_id)["availableFields"])
|
|
106
|
+
return DIMENSIONS if available_fields.empty?
|
|
107
|
+
|
|
108
|
+
# Cloudflare plans can expose different fields. Query only dimensions
|
|
109
|
+
# advertised by the zone so an optional breakdown cannot fail the import.
|
|
110
|
+
DIMENSIONS.select do |_dimension, field|
|
|
111
|
+
available_fields.any? { |available| available == field || available.end_with?("_#{field}") }
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def dataset_settings(zone_id)
|
|
116
|
+
@dataset_settings ||= {}
|
|
117
|
+
@dataset_settings[zone_id] ||= load_dataset_settings(zone_id)
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def load_dataset_settings(zone_id)
|
|
121
|
+
data = execute(
|
|
122
|
+
query: <<~GRAPHQL,
|
|
123
|
+
query TrafficSettings($zoneTag: string) {
|
|
124
|
+
viewer {
|
|
125
|
+
zones(filter: { zoneTag: $zoneTag }) {
|
|
126
|
+
settings {
|
|
127
|
+
httpRequestsAdaptiveGroups {
|
|
128
|
+
enabled
|
|
129
|
+
availableFields
|
|
130
|
+
maxPageSize
|
|
131
|
+
maxDuration
|
|
132
|
+
notOlderThan
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
}
|
|
138
|
+
GRAPHQL
|
|
139
|
+
variables: { zoneTag: zone_id }
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
zone = extract_zone(data)
|
|
143
|
+
settings = zone.dig("settings", "httpRequestsAdaptiveGroups") || {}
|
|
144
|
+
raise Error, "httpRequestsAdaptiveGroups is not enabled for this zone" if settings["enabled"] == false
|
|
145
|
+
|
|
146
|
+
settings
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
def fetch_groups(zone_id:, hostname:, date:, page_size:, dimension: nil, field: nil)
|
|
150
|
+
rows = []
|
|
151
|
+
filter = base_filter(hostname, date)
|
|
152
|
+
|
|
153
|
+
max_pages.times do
|
|
154
|
+
page = execute_group_query(
|
|
155
|
+
zone_id:,
|
|
156
|
+
filter:,
|
|
157
|
+
page_size:,
|
|
158
|
+
field:
|
|
159
|
+
)
|
|
160
|
+
rows.concat(transform_rows(page, dimension:, field:))
|
|
161
|
+
return rows if page.length < page_size
|
|
162
|
+
|
|
163
|
+
filter = base_filter(hostname, date).merge(cursor_filter(page.last, field:))
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
raise Error, "Cloudflare result exceeded the configured max_pages limit"
|
|
167
|
+
end
|
|
168
|
+
|
|
169
|
+
def execute_group_query(zone_id:, filter:, page_size:, field:)
|
|
170
|
+
dimensions = [ "clientRequestPath", field ].compact.join("\n")
|
|
171
|
+
order = [ "clientRequestPath_ASC", field && "#{field}_ASC" ].compact.join(", ")
|
|
172
|
+
query = <<~GRAPHQL
|
|
173
|
+
query TrafficGroups($zoneTag: string, $filter: filter) {
|
|
174
|
+
viewer {
|
|
175
|
+
zones(filter: { zoneTag: $zoneTag }) {
|
|
176
|
+
groups: httpRequestsAdaptiveGroups(
|
|
177
|
+
filter: $filter
|
|
178
|
+
limit: #{page_size}
|
|
179
|
+
orderBy: [#{order}]
|
|
180
|
+
) {
|
|
181
|
+
count
|
|
182
|
+
sum { visits }
|
|
183
|
+
dimensions {
|
|
184
|
+
#{dimensions}
|
|
185
|
+
}
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
}
|
|
190
|
+
GRAPHQL
|
|
191
|
+
|
|
192
|
+
data = execute(query:, variables: { zoneTag: zone_id, filter: })
|
|
193
|
+
extract_zone(data).fetch("groups", [])
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
def base_filter(hostname, date)
|
|
197
|
+
day = date.to_date
|
|
198
|
+
{
|
|
199
|
+
datetime_geq: Time.utc(day.year, day.month, day.day).iso8601,
|
|
200
|
+
datetime_lt: Time.utc(day.next_day.year, day.next_day.month, day.next_day.day).iso8601,
|
|
201
|
+
clientRequestHTTPHost: hostname,
|
|
202
|
+
requestSource: "eyeball"
|
|
203
|
+
}
|
|
204
|
+
end
|
|
205
|
+
|
|
206
|
+
# Cloudflare GraphQL has no cursor pagination. A composite cursor preserves
|
|
207
|
+
# rows that share the same path while advancing the secondary dimension.
|
|
208
|
+
def cursor_filter(row, field:)
|
|
209
|
+
path = row.dig("dimensions", "clientRequestPath")
|
|
210
|
+
return { clientRequestPath_gt: path } unless field
|
|
211
|
+
|
|
212
|
+
value = row.dig("dimensions", field).to_s
|
|
213
|
+
{
|
|
214
|
+
OR: [
|
|
215
|
+
{ clientRequestPath_gt: path },
|
|
216
|
+
{ clientRequestPath: path, "#{field}_gt" => value }
|
|
217
|
+
]
|
|
218
|
+
}
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
def transform_rows(rows, dimension:, field:)
|
|
222
|
+
rows.map do |row|
|
|
223
|
+
transformed = {
|
|
224
|
+
path: row.dig("dimensions", "clientRequestPath"),
|
|
225
|
+
page_views: Integer(row.fetch("count", 0)),
|
|
226
|
+
visits: Integer(row.dig("sum", "visits") || 0)
|
|
227
|
+
}
|
|
228
|
+
next transformed unless dimension
|
|
229
|
+
|
|
230
|
+
transformed.merge(
|
|
231
|
+
dimension: dimension.to_s,
|
|
232
|
+
value: row.dig("dimensions", field)
|
|
233
|
+
)
|
|
234
|
+
end
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
def execute(query:, variables:)
|
|
238
|
+
attempts = 0
|
|
239
|
+
|
|
240
|
+
# Permanent GraphQL and authentication errors fail immediately. Only
|
|
241
|
+
# network failures and transient provider responses are safe to retry.
|
|
242
|
+
begin
|
|
243
|
+
response = transport.call(
|
|
244
|
+
uri: ENDPOINT,
|
|
245
|
+
headers: request_headers,
|
|
246
|
+
body: JSON.generate(query:, variables:)
|
|
247
|
+
)
|
|
248
|
+
parse_response(response)
|
|
249
|
+
rescue *NETWORK_ERRORS, TransientError => error
|
|
250
|
+
raise if attempts >= max_retries
|
|
251
|
+
|
|
252
|
+
sleeper.call(retry_delay(attempts, error))
|
|
253
|
+
attempts += 1
|
|
254
|
+
retry
|
|
255
|
+
end
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
def request_headers
|
|
259
|
+
{
|
|
260
|
+
"Authorization" => "Bearer #{token}",
|
|
261
|
+
"Accept" => "application/json",
|
|
262
|
+
"Content-Type" => "application/json"
|
|
263
|
+
}
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
def parse_response(response)
|
|
267
|
+
status = Integer(response.code)
|
|
268
|
+
if TRANSIENT_HTTP_STATUSES.include?(status)
|
|
269
|
+
raise TransientError.new(
|
|
270
|
+
"Cloudflare returned HTTP #{status}",
|
|
271
|
+
retry_after: response["Retry-After"]
|
|
272
|
+
)
|
|
273
|
+
end
|
|
274
|
+
raise Error, "Cloudflare authentication failed" if [ 401, 403 ].include?(status)
|
|
275
|
+
raise Error, "Cloudflare returned HTTP #{status}" unless status.between?(200, 299)
|
|
276
|
+
|
|
277
|
+
payload = JSON.parse(response.body)
|
|
278
|
+
errors = Array(payload["errors"])
|
|
279
|
+
raise_graphql_error(errors) if errors.any?
|
|
280
|
+
|
|
281
|
+
payload.fetch("data")
|
|
282
|
+
rescue JSON::ParserError, KeyError
|
|
283
|
+
raise Error, "Cloudflare returned an invalid GraphQL response"
|
|
284
|
+
end
|
|
285
|
+
|
|
286
|
+
def raise_graphql_error(errors)
|
|
287
|
+
message = errors.filter_map { |error| error["message"] }.join("; ").presence ||
|
|
288
|
+
"Cloudflare GraphQL query failed"
|
|
289
|
+
transient = errors.any? do |error|
|
|
290
|
+
code = error.dig("extensions", "code").to_s
|
|
291
|
+
TRANSIENT_GRAPHQL_CODES.include?(code) || error["message"].to_s.match?(TRANSIENT_GRAPHQL_ERROR)
|
|
292
|
+
end
|
|
293
|
+
error_class = transient ? TransientError : Error
|
|
294
|
+
raise error_class, message
|
|
295
|
+
end
|
|
296
|
+
|
|
297
|
+
def extract_zone(data)
|
|
298
|
+
zones = data.dig("viewer", "zones")
|
|
299
|
+
raise Error, "Cloudflare response did not include the configured zone" unless zones&.one?
|
|
300
|
+
|
|
301
|
+
zones.first
|
|
302
|
+
end
|
|
303
|
+
|
|
304
|
+
def retry_delay(attempt, error)
|
|
305
|
+
retry_after = error.respond_to?(:retry_after) && error.retry_after
|
|
306
|
+
Integer(retry_after, exception: false) || (2**attempt)
|
|
307
|
+
end
|
|
308
|
+
end
|
|
309
|
+
end
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
require "uri"
|
|
2
|
+
|
|
3
|
+
module LesliAudit::Cloudflare
|
|
4
|
+
class SnapshotConverter
|
|
5
|
+
Result = Data.define(:snapshots, :ignored_paths)
|
|
6
|
+
|
|
7
|
+
def initialize(resource_resolver:)
|
|
8
|
+
@resource_resolver = resource_resolver
|
|
9
|
+
end
|
|
10
|
+
|
|
11
|
+
def call(account:, hostname:, date:, traffic_data:, snapshot_at: Time.current)
|
|
12
|
+
ignored_paths = []
|
|
13
|
+
snapshots = traffic_data.totals.filter_map do |totals|
|
|
14
|
+
path = normalize_path(totals[:path])
|
|
15
|
+
resource = resolve_resource(account:, hostname:, path:)
|
|
16
|
+
unless resource
|
|
17
|
+
ignored_paths << path
|
|
18
|
+
next
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
LesliAudit::CloudflareTrafficSnapshot.new(
|
|
22
|
+
hostname:,
|
|
23
|
+
path:,
|
|
24
|
+
resource_type: resource.fetch(:resource_type),
|
|
25
|
+
resource_id: resource.fetch(:resource_id),
|
|
26
|
+
date:,
|
|
27
|
+
page_views: totals[:page_views],
|
|
28
|
+
visits: totals[:visits],
|
|
29
|
+
snapshot_at:,
|
|
30
|
+
breakdowns: breakdowns_for(traffic_data.breakdowns, path)
|
|
31
|
+
)
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
Result.new(snapshots:, ignored_paths: ignored_paths.uniq)
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
private
|
|
38
|
+
|
|
39
|
+
attr_reader :resource_resolver
|
|
40
|
+
|
|
41
|
+
def resolve_resource(account:, hostname:, path:)
|
|
42
|
+
resource = resource_resolver.call(account:, hostname:, path:)
|
|
43
|
+
return if resource.nil?
|
|
44
|
+
|
|
45
|
+
resource.to_h.symbolize_keys
|
|
46
|
+
rescue NoMethodError
|
|
47
|
+
raise ArgumentError, "resource_resolver must return a hash or nil"
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
def breakdowns_for(rows, path)
|
|
51
|
+
# Different Cloudflare labels can normalize to the same LesliAudit value,
|
|
52
|
+
# so their totals are combined before one snapshot is created.
|
|
53
|
+
grouped = Hash.new { |hash, key| hash[key] = { page_views: 0, visits: 0 } }
|
|
54
|
+
|
|
55
|
+
rows.each do |row|
|
|
56
|
+
next unless normalize_path(row[:path]) == path
|
|
57
|
+
|
|
58
|
+
value = normalize_breakdown_value(row[:dimension], row[:value])
|
|
59
|
+
next unless value
|
|
60
|
+
|
|
61
|
+
totals = grouped[[ row[:dimension], value ]]
|
|
62
|
+
totals[:page_views] += row[:page_views]
|
|
63
|
+
totals[:visits] += row[:visits]
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
grouped.map do |(dimension, value), totals|
|
|
67
|
+
{ dimension:, value:, **totals }
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def normalize_breakdown_value(dimension, value)
|
|
72
|
+
case dimension
|
|
73
|
+
when "country"
|
|
74
|
+
country = value.to_s.strip.upcase
|
|
75
|
+
country if country.match?(/\A[A-Z0-9]{2}\z/)
|
|
76
|
+
when "device"
|
|
77
|
+
device = value.to_s.strip.downcase.presence || "unknown"
|
|
78
|
+
LesliAudit::CloudflareTrafficSnapshot::DEVICE_ALIASES.fetch(device, device)
|
|
79
|
+
when "referrer"
|
|
80
|
+
normalize_referrer(value)
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
def normalize_path(value)
|
|
85
|
+
path = value.to_s.split(/[?#]/, 2).first.to_s
|
|
86
|
+
path = "/#{path}" unless path.start_with?("/")
|
|
87
|
+
path = path.gsub(%r{/+}, "/")
|
|
88
|
+
path == "/" ? path : path.delete_suffix("/")
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def normalize_referrer(value)
|
|
92
|
+
referrer = value.to_s.strip
|
|
93
|
+
return "direct" if referrer.blank?
|
|
94
|
+
|
|
95
|
+
uri = URI.parse(referrer.match?(%r{\Ahttps?://}i) ? referrer : "//#{referrer}")
|
|
96
|
+
uri.host.to_s.downcase.delete_suffix(".").presence
|
|
97
|
+
rescue URI::InvalidURIError
|
|
98
|
+
nil
|
|
99
|
+
end
|
|
100
|
+
end
|
|
101
|
+
end
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
module LesliAudit::Cloudflare
|
|
2
|
+
class SyncService
|
|
3
|
+
class MissingAuditAccount < StandardError; end
|
|
4
|
+
|
|
5
|
+
Result = Data.define(:snapshots_count, :ignored_paths)
|
|
6
|
+
|
|
7
|
+
def initialize(configuration: Configuration.from_application, client: nil,
|
|
8
|
+
converter: nil)
|
|
9
|
+
@configuration = configuration.validate!
|
|
10
|
+
@client = client || GraphqlClient.new(
|
|
11
|
+
token: configuration.token,
|
|
12
|
+
page_size: configuration.page_size,
|
|
13
|
+
max_pages: configuration.max_pages
|
|
14
|
+
)
|
|
15
|
+
@converter = converter || SnapshotConverter.new(
|
|
16
|
+
resource_resolver: configuration.resource_resolver
|
|
17
|
+
)
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def call(account:, start_on:, end_on:)
|
|
21
|
+
start_on = start_on.to_date
|
|
22
|
+
end_on = end_on.to_date
|
|
23
|
+
raise ArgumentError, "end_on must be on or after start_on" if end_on < start_on
|
|
24
|
+
|
|
25
|
+
audit_account = LesliAudit::Account.find_by(account_id: account.id)
|
|
26
|
+
raise MissingAuditAccount, "LesliAudit is not enabled for account #{account.id}" unless audit_account
|
|
27
|
+
|
|
28
|
+
snapshots_count = 0
|
|
29
|
+
ignored_paths = []
|
|
30
|
+
|
|
31
|
+
(start_on..end_on).each do |date|
|
|
32
|
+
configuration.hostnames.each do |hostname|
|
|
33
|
+
imported = import_day(
|
|
34
|
+
account:,
|
|
35
|
+
audit_account:,
|
|
36
|
+
hostname:,
|
|
37
|
+
date:
|
|
38
|
+
)
|
|
39
|
+
snapshots_count += imported.snapshots.length
|
|
40
|
+
ignored_paths.concat(imported.ignored_paths)
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
Result.new(snapshots_count:, ignored_paths: ignored_paths.uniq)
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
private
|
|
48
|
+
|
|
49
|
+
attr_reader :client, :configuration, :converter
|
|
50
|
+
|
|
51
|
+
def import_day(account:, audit_account:, hostname:, date:)
|
|
52
|
+
traffic_data = client.traffic(
|
|
53
|
+
zone_id: configuration.zone_id,
|
|
54
|
+
hostname:,
|
|
55
|
+
date:
|
|
56
|
+
)
|
|
57
|
+
converted = converter.call(
|
|
58
|
+
account:,
|
|
59
|
+
hostname:,
|
|
60
|
+
date:,
|
|
61
|
+
traffic_data:,
|
|
62
|
+
snapshot_at: Time.current
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
# Network work is complete before opening the transaction. This keeps the
|
|
66
|
+
# lock short while making every resource for this hostname/day coherent.
|
|
67
|
+
LesliAudit::TrafficRollup.transaction do
|
|
68
|
+
converted.snapshots.each do |snapshot|
|
|
69
|
+
LesliAudit::TrafficSnapshotImporter.call(
|
|
70
|
+
account: audit_account,
|
|
71
|
+
snapshot:
|
|
72
|
+
)
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
converted
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
end
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
require "uri"
|
|
2
|
+
|
|
3
|
+
module LesliAudit
|
|
4
|
+
# Immutable input contract for one complete Cloudflare traffic snapshot.
|
|
5
|
+
# Query strings, fragments, credentials, and referrer paths are discarded
|
|
6
|
+
# before any value reaches the database.
|
|
7
|
+
class CloudflareTrafficSnapshot
|
|
8
|
+
BREAKDOWNS_OMITTED = Object.new.freeze
|
|
9
|
+
PROVIDER = "cloudflare".freeze
|
|
10
|
+
MAX_IDENTIFIER_LENGTH = 255
|
|
11
|
+
|
|
12
|
+
DEVICE_ALIASES = {
|
|
13
|
+
"phone" => "mobile",
|
|
14
|
+
"smartphone" => "mobile",
|
|
15
|
+
"pc" => "desktop"
|
|
16
|
+
}.freeze
|
|
17
|
+
|
|
18
|
+
attr_reader :breakdowns, :date, :hostname, :page_views, :path,
|
|
19
|
+
:resource_id, :resource_type, :snapshot_at, :visits
|
|
20
|
+
|
|
21
|
+
def initialize(hostname:, path:, resource_type:, resource_id:, date:,
|
|
22
|
+
page_views:, visits:, snapshot_at:, breakdowns: BREAKDOWNS_OMITTED)
|
|
23
|
+
@hostname = normalize_hostname(hostname).freeze
|
|
24
|
+
@path = normalize_path(path).freeze
|
|
25
|
+
@resource_type = normalize_text(resource_type, :resource_type).freeze
|
|
26
|
+
@resource_id = normalize_positive_integer(resource_id, :resource_id)
|
|
27
|
+
@date = normalize_date(date)
|
|
28
|
+
@page_views = normalize_count(page_views, :page_views)
|
|
29
|
+
@visits = normalize_count(visits, :visits)
|
|
30
|
+
@snapshot_at = normalize_time(snapshot_at)
|
|
31
|
+
@breakdowns_provided = !breakdowns.equal?(BREAKDOWNS_OMITTED)
|
|
32
|
+
@breakdowns = @breakdowns_provided ? normalize_breakdowns(breakdowns) : [].freeze
|
|
33
|
+
freeze
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def provider
|
|
37
|
+
PROVIDER
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def breakdowns_provided?
|
|
41
|
+
@breakdowns_provided
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
private
|
|
45
|
+
|
|
46
|
+
def normalize_hostname(value)
|
|
47
|
+
raw_hostname = normalize_text(value, :hostname)
|
|
48
|
+
uri = URI.parse(raw_hostname.match?(%r{\Ahttps?://}i) ? raw_hostname : "//#{raw_hostname}")
|
|
49
|
+
raise ArgumentError, "hostname must not include credentials" if uri.userinfo.present?
|
|
50
|
+
|
|
51
|
+
hostname = uri.host.to_s.downcase.delete_suffix(".")
|
|
52
|
+
valid = hostname.present? && hostname.length <= MAX_IDENTIFIER_LENGTH
|
|
53
|
+
valid &&= hostname.match?(/\A[a-z0-9](?:[a-z0-9.-]*[a-z0-9])?\z/)
|
|
54
|
+
valid &&= !hostname.include?("..")
|
|
55
|
+
raise ArgumentError, "hostname must be a valid DNS hostname" unless valid
|
|
56
|
+
|
|
57
|
+
hostname
|
|
58
|
+
rescue URI::InvalidURIError
|
|
59
|
+
raise ArgumentError, "hostname must be a valid DNS hostname"
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def normalize_path(value)
|
|
63
|
+
raw_path = normalize_text(value, :path)
|
|
64
|
+
raw_path = URI.parse(raw_path).path if raw_path.match?(%r{\Ahttps?://}i)
|
|
65
|
+
path = raw_path.to_s.split(/[?#]/, 2).first.to_s
|
|
66
|
+
path = "/#{path}" unless path.start_with?("/")
|
|
67
|
+
path = path.gsub(%r{/+}, "/")
|
|
68
|
+
path = path.delete_suffix("/") unless path == "/"
|
|
69
|
+
|
|
70
|
+
valid = path.length <= MAX_IDENTIFIER_LENGTH && !path.match?(/[[:cntrl:]]/)
|
|
71
|
+
raise ArgumentError, "path must be a valid URL path" unless valid
|
|
72
|
+
|
|
73
|
+
path
|
|
74
|
+
rescue URI::InvalidURIError
|
|
75
|
+
raise ArgumentError, "path must be a valid URL path"
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def normalize_breakdowns(value)
|
|
79
|
+
rows = Array.wrap(value).map do |attributes|
|
|
80
|
+
attributes = attributes.to_h.symbolize_keys
|
|
81
|
+
dimension = attributes.fetch(:dimension).to_s.strip.downcase
|
|
82
|
+
{
|
|
83
|
+
dimension: dimension.freeze,
|
|
84
|
+
value: normalize_breakdown_value(dimension, attributes.fetch(:value)).freeze,
|
|
85
|
+
page_views: normalize_count(attributes.fetch(:page_views), :page_views),
|
|
86
|
+
visits: normalize_count(attributes.fetch(:visits), :visits)
|
|
87
|
+
}.freeze
|
|
88
|
+
rescue KeyError, NoMethodError
|
|
89
|
+
raise ArgumentError, "each breakdown must include dimension, value, page_views, and visits"
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
keys = rows.map { |row| [ row[:dimension], row[:value] ] }
|
|
93
|
+
raise ArgumentError, "breakdowns must be unique after normalization" if keys.uniq.length != keys.length
|
|
94
|
+
|
|
95
|
+
rows.freeze
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def normalize_breakdown_value(dimension, value)
|
|
99
|
+
unless TrafficBreakdown::DIMENSIONS.include?(dimension)
|
|
100
|
+
raise ArgumentError, "unsupported breakdown dimension: #{dimension}"
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
case dimension
|
|
104
|
+
when "country"
|
|
105
|
+
normalize_country(value)
|
|
106
|
+
when "device"
|
|
107
|
+
normalize_device(value)
|
|
108
|
+
when "referrer"
|
|
109
|
+
normalize_referrer(value)
|
|
110
|
+
else
|
|
111
|
+
normalize_label(value, dimension)
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def normalize_country(value)
|
|
116
|
+
country = normalize_label(value, :country).upcase
|
|
117
|
+
raise ArgumentError, "country must be a two-character code" unless country.match?(/\A[A-Z0-9]{2}\z/)
|
|
118
|
+
|
|
119
|
+
country
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
def normalize_device(value)
|
|
123
|
+
device = normalize_label(value, :device)
|
|
124
|
+
DEVICE_ALIASES.fetch(device, device)
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def normalize_referrer(value)
|
|
128
|
+
referrer = value.to_s.strip
|
|
129
|
+
return "direct" if referrer.blank? || %w[(direct) direct none].include?(referrer.downcase)
|
|
130
|
+
|
|
131
|
+
uri = URI.parse(referrer.match?(%r{\Ahttps?://}i) ? referrer : "//#{referrer}")
|
|
132
|
+
return normalize_hostname(uri.host) if uri.host.present?
|
|
133
|
+
|
|
134
|
+
normalize_label(referrer.split(%r{[/?#]}, 2).first, :referrer)
|
|
135
|
+
rescue URI::InvalidURIError
|
|
136
|
+
raise ArgumentError, "referrer must be a hostname or HTTP URL"
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def normalize_label(value, name)
|
|
140
|
+
normalize_text(value, name).squish.downcase
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def normalize_text(value, name)
|
|
144
|
+
text = value.to_s.strip
|
|
145
|
+
valid = text.present? && text.length <= MAX_IDENTIFIER_LENGTH
|
|
146
|
+
valid &&= !text.match?(/[[:cntrl:]]/)
|
|
147
|
+
raise ArgumentError, "#{name} must be present" unless valid
|
|
148
|
+
|
|
149
|
+
text
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
def normalize_positive_integer(value, name)
|
|
153
|
+
number = Integer(value)
|
|
154
|
+
raise ArgumentError if number <= 0
|
|
155
|
+
|
|
156
|
+
number
|
|
157
|
+
rescue ArgumentError, TypeError
|
|
158
|
+
raise ArgumentError, "#{name} must be a positive integer"
|
|
159
|
+
end
|
|
160
|
+
|
|
161
|
+
def normalize_count(value, name)
|
|
162
|
+
count = Integer(value)
|
|
163
|
+
raise ArgumentError if count.negative?
|
|
164
|
+
|
|
165
|
+
count
|
|
166
|
+
rescue ArgumentError, TypeError
|
|
167
|
+
raise ArgumentError, "#{name} must be a nonnegative integer"
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
def normalize_date(value)
|
|
171
|
+
value.to_date
|
|
172
|
+
rescue NoMethodError, Date::Error
|
|
173
|
+
raise ArgumentError, "date must be date-like"
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
def normalize_time(value)
|
|
177
|
+
value.in_time_zone
|
|
178
|
+
rescue NoMethodError, ArgumentError
|
|
179
|
+
raise ArgumentError, "snapshot_at must be time-like"
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
end
|