lesli_audit 1.1.2 → 1.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/app/assets/stylesheets/lesli_audit/application.css +183 -0
- data/app/controllers/lesli_audit/analytics_controller.rb +16 -0
- data/app/controllers/lesli_audit/logs_controller.rb +12 -2
- data/app/controllers/lesli_audit/traffic_controller.rb +25 -0
- data/app/controllers/lesli_audit/users_controller.rb +7 -4
- data/app/interfaces/lesli_audit/logger_interface.rb +5 -4
- data/app/jobs/lesli_audit/cloudflare_traffic_sync_job.rb +78 -0
- data/app/models/lesli_audit/account.rb +12 -0
- data/app/models/lesli_audit/cloudflare_traffic_sync.rb +99 -0
- data/app/models/lesli_audit/traffic_breakdown.rb +38 -0
- data/app/models/lesli_audit/traffic_rollup.rb +33 -0
- data/app/models/lesli_audit/user_journal.rb +0 -2
- data/app/services/lesli_audit/activity_audit.rb +8 -4
- data/app/services/lesli_audit/analytics_service.rb +237 -0
- data/app/services/lesli_audit/cloudflare/configuration.rb +92 -0
- data/app/services/lesli_audit/cloudflare/graphql_client.rb +309 -0
- data/app/services/lesli_audit/cloudflare/snapshot_converter.rb +101 -0
- data/app/services/lesli_audit/cloudflare/sync_service.rb +79 -0
- data/app/services/lesli_audit/cloudflare_traffic_snapshot.rb +182 -0
- data/app/services/lesli_audit/log_service.rb +99 -5
- data/app/services/lesli_audit/request_service.rb +73 -10
- data/app/services/lesli_audit/retention_cleanup.rb +16 -16
- data/app/services/lesli_audit/traffic_service.rb +216 -0
- data/app/services/lesli_audit/traffic_snapshot_importer.rb +120 -0
- data/app/services/lesli_audit/user_service.rb +119 -152
- data/app/views/lesli_audit/{visitors → analytics}/index.html.erb +40 -30
- data/app/views/lesli_audit/logs/index.html.erb +88 -19
- data/app/views/lesli_audit/logs/show.html.erb +36 -8
- data/app/views/lesli_audit/partials/_navigation.html.erb +5 -4
- data/app/views/lesli_audit/requests/index.html.erb +80 -19
- data/app/views/lesli_audit/traffic/_breakdown.html.erb +46 -0
- data/app/views/lesli_audit/traffic/index.html.erb +177 -0
- data/app/views/lesli_audit/users/index.html.erb +97 -40
- data/config/locales/translations.en.yml +137 -1
- data/config/locales/translations.es.yml +137 -1
- data/config/locales/translations.fr.yml +132 -1
- data/config/locales/translations.it.yml +132 -1
- data/config/locales/translations.pt.yml +132 -1
- data/config/routes.rb +6 -23
- data/db/migrate/v1.0/0501110110_create_lesli_audit_account_logs.rb +2 -1
- data/db/migrate/v1.0/0501120110_create_lesli_audit_user_logs.rb +2 -1
- data/db/migrate/v1.0/0501120210_create_lesli_audit_user_journals.rb +0 -3
- data/db/migrate/v1.0/0501130110_create_lesli_audit_traffic_rollups.rb +55 -0
- data/db/migrate/v1.0/0501130210_create_lesli_audit_traffic_breakdowns.rb +41 -0
- data/db/migrate/v1.0/0501130310_create_lesli_audit_cloudflare_traffic_syncs.rb +42 -0
- data/db/seed/devices.rb +2 -1
- data/db/seed/traffic.rb +128 -0
- data/db/seed/users.rb +68 -27
- data/db/seeds.rb +3 -2
- data/lib/lesli_audit/version.rb +2 -2
- data/lib/tasks/lesli_audit_cloudflare_traffic.rake +16 -0
- data/readme.md +10 -0
- metadata +24 -5
- data/app/controllers/lesli_audit/visitors_controller.rb +0 -14
- data/app/services/lesli_audit/visitor_service.rb +0 -212
|
@@ -27,17 +27,111 @@ Building a better future, one line of code at a time.
|
|
|
27
27
|
@license GPLv3 http://www.gnu.org/licenses/gpl-3.0.en.html
|
|
28
28
|
|
|
29
29
|
// · ~·~ ~·~ ~·~ ~·~ ~·~ ~·~ ~·~ ~·~ ~·~
|
|
30
|
-
// ·
|
|
30
|
+
// ·
|
|
31
31
|
=end
|
|
32
32
|
|
|
33
33
|
module LesliAudit
|
|
34
34
|
class LogService < Lesli::ApplicationLesliService
|
|
35
|
+
LOG_KINDS = %w[user account].freeze
|
|
36
|
+
DEFAULT_KIND = "user"
|
|
37
|
+
DEFAULT_PER_PAGE = 12
|
|
38
|
+
MAX_PER_PAGE = 100
|
|
39
|
+
|
|
40
|
+
SEARCH_FIELDS = %i[
|
|
41
|
+
operation
|
|
42
|
+
description
|
|
43
|
+
source
|
|
44
|
+
action
|
|
45
|
+
actor_uid
|
|
46
|
+
subject_type
|
|
47
|
+
].freeze
|
|
35
48
|
|
|
36
49
|
def index
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
50
|
+
logs = apply_search(logs_scope)
|
|
51
|
+
|
|
52
|
+
logs
|
|
53
|
+
.order(sort_order, id: :desc)
|
|
54
|
+
.page(page)
|
|
55
|
+
.per(per_page)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def find(id)
|
|
59
|
+
logs_scope.find(id)
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def kind
|
|
63
|
+
requested_kind = query[:kind].to_s
|
|
64
|
+
LOG_KINDS.include?(requested_kind) ? requested_kind : DEFAULT_KIND
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
def available_kinds
|
|
68
|
+
LOG_KINDS
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
private
|
|
72
|
+
|
|
73
|
+
def logs_scope
|
|
74
|
+
return account_logs if kind == "account"
|
|
75
|
+
|
|
76
|
+
user_logs
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def account_logs
|
|
80
|
+
audit_account.account_logs
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def user_logs
|
|
84
|
+
audit_account.user_logs
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def audit_account
|
|
88
|
+
current_user.account.audit
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def apply_search(logs)
|
|
92
|
+
term = query[:search].to_s.strip
|
|
93
|
+
return logs if term.blank?
|
|
94
|
+
|
|
95
|
+
# Build one case-insensitive condition from a fixed field allowlist;
|
|
96
|
+
# the search text remains a bound value instead of becoming SQL.
|
|
97
|
+
pattern = "%#{ActiveRecord::Base.sanitize_sql_like(term.downcase)}%"
|
|
98
|
+
conditions = SEARCH_FIELDS.map do |field|
|
|
99
|
+
lower(log_table[field]).matches(pattern)
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
logs.where(conditions.reduce(:or))
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def lower(attribute)
|
|
106
|
+
Arel::Nodes::NamedFunction.new("LOWER", [ attribute ])
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def sort_order
|
|
110
|
+
direction = query.dig(:order, :dir).to_s.casecmp?("asc") ? :asc : :desc
|
|
111
|
+
sort_column.public_send(direction)
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def sort_column
|
|
115
|
+
field = query.dig(:order, :by).to_s
|
|
116
|
+
field = "created_at" unless %w[operation source actor_uid created_at].include?(field)
|
|
117
|
+
|
|
118
|
+
log_table[field]
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def log_table
|
|
122
|
+
kind == "account" ? AccountLog.arel_table : UserLog.arel_table
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def page
|
|
126
|
+
requested_page = query.dig(:pagination, :page).to_i
|
|
127
|
+
requested_page.positive? ? requested_page : 1
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def per_page
|
|
131
|
+
requested_size = query.dig(:pagination, :perPage).to_i
|
|
132
|
+
return DEFAULT_PER_PAGE unless requested_size.positive?
|
|
133
|
+
|
|
134
|
+
[ requested_size, MAX_PER_PAGE ].min
|
|
41
135
|
end
|
|
42
136
|
end
|
|
43
137
|
end
|
|
@@ -27,22 +27,85 @@ Building a better future, one line of code at a time.
|
|
|
27
27
|
@license GPLv3 http://www.gnu.org/licenses/gpl-3.0.en.html
|
|
28
28
|
|
|
29
29
|
// · ~·~ ~·~ ~·~ ~·~ ~·~ ~·~ ~·~ ~·~ ~·~
|
|
30
|
-
// ·
|
|
30
|
+
// ·
|
|
31
31
|
=end
|
|
32
32
|
|
|
33
33
|
module LesliAudit
|
|
34
34
|
class RequestService < Lesli::ApplicationLesliService
|
|
35
|
+
ACCOUNT_WIDE_ROLES = %w[owner admin].freeze
|
|
36
|
+
DEFAULT_PER_PAGE = 12
|
|
37
|
+
MAX_PER_PAGE = 100
|
|
38
|
+
|
|
39
|
+
SORT_COLUMNS = {
|
|
40
|
+
"user" => Lesli::User.arel_table[:email],
|
|
41
|
+
"controller" => UserRequest.arel_table[:request_controller],
|
|
42
|
+
"action" => UserRequest.arel_table[:request_action],
|
|
43
|
+
"count" => UserRequest.arel_table[:request_count],
|
|
44
|
+
"date" => UserRequest.arel_table[:date]
|
|
45
|
+
}.freeze
|
|
35
46
|
|
|
36
|
-
# @overwrite
|
|
37
|
-
# @return {Hash} Paginated list of the records
|
|
38
|
-
# @param {query} Has of the formated queries/filters that will be applied to filter data
|
|
39
|
-
# @description
|
|
40
|
-
# @example
|
|
41
47
|
def index
|
|
42
|
-
current_user.
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
48
|
+
requests = current_user.account.audit.user_requests
|
|
49
|
+
.joins(:user)
|
|
50
|
+
.select(
|
|
51
|
+
"lesli_audit_user_requests.*",
|
|
52
|
+
"lesli_users.email AS user_email"
|
|
53
|
+
)
|
|
54
|
+
|
|
55
|
+
requests = requests.where(user_id: current_user.id) unless account_wide?
|
|
56
|
+
requests = apply_search(requests)
|
|
57
|
+
|
|
58
|
+
requests
|
|
59
|
+
.order(sort_order, id: :desc)
|
|
60
|
+
.page(page)
|
|
61
|
+
.per(per_page)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
private
|
|
65
|
+
|
|
66
|
+
# LesliShield authorizes access to the controller. Built-in owner and
|
|
67
|
+
# admin roles additionally receive the account-wide reporting scope.
|
|
68
|
+
def account_wide?
|
|
69
|
+
current_user.respond_to?(:has_roles?) &&
|
|
70
|
+
current_user.has_roles?(*ACCOUNT_WIDE_ROLES)
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def apply_search(requests)
|
|
74
|
+
term = query[:search].to_s.strip
|
|
75
|
+
return requests if term.blank?
|
|
76
|
+
|
|
77
|
+
pattern = "%#{ActiveRecord::Base.sanitize_sql_like(term.downcase)}%"
|
|
78
|
+
|
|
79
|
+
requests.where(
|
|
80
|
+
<<~SQL.squish,
|
|
81
|
+
LOWER(lesli_users.email) LIKE :pattern
|
|
82
|
+
OR LOWER(lesli_audit_user_requests.request_controller) LIKE :pattern
|
|
83
|
+
OR LOWER(lesli_audit_user_requests.request_action) LIKE :pattern
|
|
84
|
+
SQL
|
|
85
|
+
pattern: pattern
|
|
86
|
+
)
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def sort_column
|
|
90
|
+
SORT_COLUMNS.fetch(query.dig(:order, :by).to_s, SORT_COLUMNS.fetch("date"))
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def sort_order
|
|
94
|
+
return sort_column.asc if query.dig(:order, :dir).to_s.casecmp?("asc")
|
|
95
|
+
|
|
96
|
+
sort_column.desc
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def page
|
|
100
|
+
requested_page = query.dig(:pagination, :page).to_i
|
|
101
|
+
requested_page.positive? ? requested_page : 1
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def per_page
|
|
105
|
+
requested_size = query.dig(:pagination, :perPage).to_i
|
|
106
|
+
return DEFAULT_PER_PAGE unless requested_size.positive?
|
|
107
|
+
|
|
108
|
+
[ requested_size, MAX_PER_PAGE ].min
|
|
46
109
|
end
|
|
47
110
|
end
|
|
48
111
|
end
|
|
@@ -42,9 +42,14 @@ module LesliAudit
|
|
|
42
42
|
ANALYTICS_MODELS = [
|
|
43
43
|
LesliAudit::AccountDevice,
|
|
44
44
|
LesliAudit::AccountRequest,
|
|
45
|
+
LesliAudit::UserJournal,
|
|
45
46
|
LesliAudit::UserRequest
|
|
46
47
|
].freeze
|
|
47
48
|
|
|
49
|
+
TRAFFIC_MODELS = [
|
|
50
|
+
LesliAudit::TrafficRollup
|
|
51
|
+
].freeze
|
|
52
|
+
|
|
48
53
|
ACTIVITY_MODELS = [
|
|
49
54
|
LesliAudit::AccountLog,
|
|
50
55
|
LesliAudit::UserLog
|
|
@@ -63,13 +68,23 @@ module LesliAudit
|
|
|
63
68
|
def call
|
|
64
69
|
deleted = {}
|
|
65
70
|
|
|
66
|
-
cleanup_journals(deleted)
|
|
67
71
|
cleanup_group(
|
|
68
72
|
deleted,
|
|
69
73
|
ANALYTICS_MODELS,
|
|
70
74
|
column: :date,
|
|
71
75
|
cutoff: date_cutoff(:analytics_retention_days)
|
|
72
76
|
)
|
|
77
|
+
cleanup_group(
|
|
78
|
+
deleted,
|
|
79
|
+
TRAFFIC_MODELS,
|
|
80
|
+
column: :date,
|
|
81
|
+
cutoff: date_cutoff(:traffic_retention_days)
|
|
82
|
+
)
|
|
83
|
+
deleted[LesliAudit::CloudflareTrafficSync.table_name] = delete_with_cutoff(
|
|
84
|
+
LesliAudit::CloudflareTrafficSync.all,
|
|
85
|
+
:period_end,
|
|
86
|
+
date_cutoff(:traffic_retention_days)
|
|
87
|
+
)
|
|
73
88
|
cleanup_group(
|
|
74
89
|
deleted,
|
|
75
90
|
ACTIVITY_MODELS,
|
|
@@ -84,21 +99,6 @@ module LesliAudit
|
|
|
84
99
|
|
|
85
100
|
attr_reader :batch_size, :now, :today
|
|
86
101
|
|
|
87
|
-
def cleanup_journals(deleted)
|
|
88
|
-
retained_count = delete_with_cutoff(
|
|
89
|
-
LesliAudit::UserJournal.retained.processed,
|
|
90
|
-
:date,
|
|
91
|
-
date_cutoff(:journal_retention_days)
|
|
92
|
-
)
|
|
93
|
-
transient_count = delete_with_cutoff(
|
|
94
|
-
LesliAudit::UserJournal.transient.processed,
|
|
95
|
-
:date,
|
|
96
|
-
date_cutoff(:analytics_retention_days)
|
|
97
|
-
)
|
|
98
|
-
|
|
99
|
-
deleted[LesliAudit::UserJournal.table_name] = retained_count + transient_count
|
|
100
|
-
end
|
|
101
|
-
|
|
102
102
|
def cleanup_group(deleted, models, column:, cutoff:)
|
|
103
103
|
models.each do |model|
|
|
104
104
|
deleted[model.table_name] = delete_with_cutoff(model.all, column, cutoff)
|
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
module LesliAudit
|
|
2
|
+
class TrafficService < Lesli::ApplicationLesliService
|
|
3
|
+
LIMIT = 10
|
|
4
|
+
STALE_AFTER = 36.hours
|
|
5
|
+
|
|
6
|
+
def dashboard
|
|
7
|
+
{
|
|
8
|
+
summary: summary,
|
|
9
|
+
trend: trend,
|
|
10
|
+
paths: paths,
|
|
11
|
+
resources: resources,
|
|
12
|
+
countries: breakdown(:country),
|
|
13
|
+
devices: breakdown(:device),
|
|
14
|
+
referrers: breakdown(:referrer),
|
|
15
|
+
synchronization: synchronization
|
|
16
|
+
}
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def summary
|
|
20
|
+
page_views, visits = rollups.pick(
|
|
21
|
+
Arel.sql("COALESCE(SUM(page_views), 0)"),
|
|
22
|
+
Arel.sql("COALESCE(SUM(visits), 0)")
|
|
23
|
+
)
|
|
24
|
+
|
|
25
|
+
{
|
|
26
|
+
page_views: page_views.to_i,
|
|
27
|
+
visits: visits.to_i,
|
|
28
|
+
resources: rollups.group(:resource_type, :resource_id).count.length,
|
|
29
|
+
tracked_paths: rollups.group(:provider, :hostname, :path).count.length
|
|
30
|
+
}
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def trend
|
|
34
|
+
rollups
|
|
35
|
+
.group(:date)
|
|
36
|
+
.order(:date)
|
|
37
|
+
.pluck(
|
|
38
|
+
:date,
|
|
39
|
+
Arel.sql("SUM(page_views)"),
|
|
40
|
+
Arel.sql("SUM(visits)")
|
|
41
|
+
)
|
|
42
|
+
.map do |date, page_views, visits|
|
|
43
|
+
{
|
|
44
|
+
date:,
|
|
45
|
+
page_views: page_views.to_i,
|
|
46
|
+
visits: visits.to_i
|
|
47
|
+
}
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def resources
|
|
52
|
+
rollups
|
|
53
|
+
.group(:resource_type, :resource_id)
|
|
54
|
+
.order(Arel.sql("SUM(page_views) DESC"))
|
|
55
|
+
.limit(LIMIT)
|
|
56
|
+
.pluck(
|
|
57
|
+
:resource_type,
|
|
58
|
+
:resource_id,
|
|
59
|
+
Arel.sql("SUM(page_views)"),
|
|
60
|
+
Arel.sql("SUM(visits)")
|
|
61
|
+
)
|
|
62
|
+
.map do |resource_type, resource_id, page_views, visits|
|
|
63
|
+
{
|
|
64
|
+
"resource" => "#{resource_type} ##{resource_id}",
|
|
65
|
+
"page_views" => page_views.to_i,
|
|
66
|
+
"visits" => visits.to_i
|
|
67
|
+
}
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def paths
|
|
72
|
+
# Hostname and provider remain part of the group so equal paths
|
|
73
|
+
# from different sites or providers are never combined.
|
|
74
|
+
rollups
|
|
75
|
+
.group(:provider, :hostname, :path)
|
|
76
|
+
.order(Arel.sql("SUM(visits) DESC"))
|
|
77
|
+
.limit(LIMIT)
|
|
78
|
+
.pluck(
|
|
79
|
+
:provider,
|
|
80
|
+
:hostname,
|
|
81
|
+
:path,
|
|
82
|
+
Arel.sql("SUM(page_views)"),
|
|
83
|
+
Arel.sql("SUM(visits)")
|
|
84
|
+
)
|
|
85
|
+
.map do |provider, hostname, path, page_views, visits|
|
|
86
|
+
{
|
|
87
|
+
"provider" => provider.to_s.titleize,
|
|
88
|
+
"hostname" => hostname,
|
|
89
|
+
"path" => path,
|
|
90
|
+
"page_views" => page_views.to_i,
|
|
91
|
+
"visits" => visits.to_i
|
|
92
|
+
}
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def breakdown(dimension)
|
|
97
|
+
dimension = dimension.to_s
|
|
98
|
+
return [] unless account && TrafficBreakdown::DIMENSIONS.include?(dimension)
|
|
99
|
+
|
|
100
|
+
relation = account.traffic_breakdowns
|
|
101
|
+
.where(traffic_rollup_id: rollups.select(:id), dimension:)
|
|
102
|
+
|
|
103
|
+
total_page_views, total_visits = relation.pick(
|
|
104
|
+
Arel.sql("COALESCE(SUM(page_views), 0)"),
|
|
105
|
+
Arel.sql("COALESCE(SUM(visits), 0)")
|
|
106
|
+
)
|
|
107
|
+
records = relation
|
|
108
|
+
.group(:value)
|
|
109
|
+
.order(Arel.sql("SUM(page_views) DESC"))
|
|
110
|
+
.limit(LIMIT)
|
|
111
|
+
.pluck(
|
|
112
|
+
:value,
|
|
113
|
+
Arel.sql("SUM(page_views)"),
|
|
114
|
+
Arel.sql("SUM(visits)")
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
rows = records.map do |value, page_views, visits|
|
|
118
|
+
page_views = page_views.to_i
|
|
119
|
+
{
|
|
120
|
+
value:,
|
|
121
|
+
page_views:,
|
|
122
|
+
visits: visits.to_i,
|
|
123
|
+
percentage: percentage(page_views, total_page_views)
|
|
124
|
+
}
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
append_other(rows, total_page_views.to_i, total_visits.to_i)
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
# Synchronization is intentionally based on every rollup in the account,
|
|
131
|
+
# not the selected report filters. Filters must not make fresh data look old.
|
|
132
|
+
def synchronization(current_time: Time.current)
|
|
133
|
+
status = synchronization_status
|
|
134
|
+
last_synced_at = status[:last_synced_at]
|
|
135
|
+
|
|
136
|
+
{
|
|
137
|
+
source: status[:source].to_s.titleize.presence || "Cloudflare",
|
|
138
|
+
last_synced_at:,
|
|
139
|
+
stale: last_synced_at.present? && last_synced_at < STALE_AFTER.ago(current_time)
|
|
140
|
+
}
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
private
|
|
144
|
+
|
|
145
|
+
def account
|
|
146
|
+
@account ||= current_user&.account&.audit
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
def account_rollups
|
|
150
|
+
account ? account.traffic_rollups : TrafficRollup.none
|
|
151
|
+
end
|
|
152
|
+
|
|
153
|
+
def synchronization_status
|
|
154
|
+
if account
|
|
155
|
+
status = CloudflareTrafficSync.latest_status_for(account)
|
|
156
|
+
return status if status[:last_synced_at]
|
|
157
|
+
end
|
|
158
|
+
|
|
159
|
+
provider, last_synced_at = account_rollups
|
|
160
|
+
.order(snapshot_at: :desc)
|
|
161
|
+
.pick(:provider, :snapshot_at)
|
|
162
|
+
|
|
163
|
+
{ source: provider || CloudflareTrafficSync::SOURCE, last_synced_at: }
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def rollups
|
|
167
|
+
@rollups ||= begin
|
|
168
|
+
relation = account_rollups
|
|
169
|
+
relation = apply_time_filter(relation)
|
|
170
|
+
relation = relation.where(resource_type: query[:resource_type].to_s.strip) if query[:resource_type].present?
|
|
171
|
+
relation = relation.where(resource_id: resource_id_filter) if resource_id_filter
|
|
172
|
+
relation
|
|
173
|
+
end
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
def resource_id_filter
|
|
177
|
+
return @resource_id_filter if defined?(@resource_id_filter)
|
|
178
|
+
|
|
179
|
+
@resource_id_filter = Integer(query[:resource_id], exception: false)
|
|
180
|
+
@resource_id_filter = nil unless @resource_id_filter&.positive?
|
|
181
|
+
@resource_id_filter
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
def apply_time_filter(relation)
|
|
185
|
+
start_date = case query[:bytime].presence || "month"
|
|
186
|
+
when "day" then Date.current
|
|
187
|
+
when "week" then 6.days.ago.to_date
|
|
188
|
+
when "month" then 29.days.ago.to_date
|
|
189
|
+
when "sixmonth" then 6.months.ago.to_date
|
|
190
|
+
when "year" then 1.year.ago.to_date
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
start_date ? relation.where(date: start_date..) : relation
|
|
194
|
+
end
|
|
195
|
+
|
|
196
|
+
def percentage(count, total)
|
|
197
|
+
return 0.0 if total.to_i.zero?
|
|
198
|
+
|
|
199
|
+
((count.to_f / total) * 100).round(1)
|
|
200
|
+
end
|
|
201
|
+
|
|
202
|
+
def append_other(rows, total_page_views, total_visits)
|
|
203
|
+
other_page_views = total_page_views - rows.sum { |row| row[:page_views] }
|
|
204
|
+
other_visits = total_visits - rows.sum { |row| row[:visits] }
|
|
205
|
+
return rows unless other_page_views.positive? || other_visits.positive?
|
|
206
|
+
|
|
207
|
+
rows << {
|
|
208
|
+
value: "other",
|
|
209
|
+
page_views: other_page_views,
|
|
210
|
+
visits: other_visits,
|
|
211
|
+
percentage: percentage(other_page_views, total_page_views),
|
|
212
|
+
other: true
|
|
213
|
+
}
|
|
214
|
+
end
|
|
215
|
+
end
|
|
216
|
+
end
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
module LesliAudit
|
|
2
|
+
# Persists one complete set of totals for a Cloudflare path and day.
|
|
3
|
+
# A newer snapshot replaces the totals; retries and older snapshots are
|
|
4
|
+
# no-ops. Breakdowns are replaced only when the DTO explicitly provides
|
|
5
|
+
# them, allowing omitted data and an intentionally empty set to differ.
|
|
6
|
+
class TrafficSnapshotImporter
|
|
7
|
+
UNIQUE_INDEX = "index_lesli_audit_traffic_rollups_on_provider_path_and_date"
|
|
8
|
+
BREAKDOWN_INDEX = "index_lesli_audit_traffic_breakdowns_on_dimension"
|
|
9
|
+
|
|
10
|
+
class ResourceConflict < StandardError; end
|
|
11
|
+
|
|
12
|
+
def self.call(account:, snapshot:)
|
|
13
|
+
new(account:, snapshot:).call
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def initialize(account:, snapshot:)
|
|
17
|
+
@account = account
|
|
18
|
+
@snapshot = snapshot
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def call
|
|
22
|
+
validate_input!
|
|
23
|
+
|
|
24
|
+
# The account row is the stable lock shared by new and existing
|
|
25
|
+
# paths, so concurrent imports cannot race before a rollup exists.
|
|
26
|
+
account.with_lock do
|
|
27
|
+
rollup = locked_rollup
|
|
28
|
+
ensure_resource_matches!(rollup) if rollup
|
|
29
|
+
return rollup if rollup && rollup.snapshot_at >= snapshot.snapshot_at
|
|
30
|
+
|
|
31
|
+
upsert_rollup
|
|
32
|
+
rollup = locked_rollup
|
|
33
|
+
replace_breakdowns(rollup) if snapshot.breakdowns_provided?
|
|
34
|
+
rollup.reload
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
private
|
|
39
|
+
|
|
40
|
+
attr_reader :account, :snapshot
|
|
41
|
+
|
|
42
|
+
def validate_input!
|
|
43
|
+
unless account.is_a?(LesliAudit::Account) && account.persisted?
|
|
44
|
+
raise ArgumentError, "account must be a persisted LesliAudit::Account"
|
|
45
|
+
end
|
|
46
|
+
return if snapshot.is_a?(LesliAudit::CloudflareTrafficSnapshot)
|
|
47
|
+
|
|
48
|
+
raise ArgumentError, "snapshot must be a CloudflareTrafficSnapshot"
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def identity
|
|
52
|
+
{
|
|
53
|
+
provider: snapshot.provider,
|
|
54
|
+
hostname: snapshot.hostname,
|
|
55
|
+
path: snapshot.path,
|
|
56
|
+
date: snapshot.date
|
|
57
|
+
}
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def locked_rollup
|
|
61
|
+
account.traffic_rollups.lock.find_by(identity)
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def upsert_rollup
|
|
65
|
+
now = Time.current
|
|
66
|
+
TrafficRollup.upsert_all(
|
|
67
|
+
[ {
|
|
68
|
+
account_id: account.id,
|
|
69
|
+
**identity,
|
|
70
|
+
resource_type: snapshot.resource_type,
|
|
71
|
+
resource_id: snapshot.resource_id,
|
|
72
|
+
page_views: snapshot.page_views,
|
|
73
|
+
visits: snapshot.visits,
|
|
74
|
+
snapshot_at: snapshot.snapshot_at,
|
|
75
|
+
created_at: now,
|
|
76
|
+
updated_at: now
|
|
77
|
+
} ],
|
|
78
|
+
unique_by: UNIQUE_INDEX,
|
|
79
|
+
update_only: %i[page_views visits snapshot_at updated_at]
|
|
80
|
+
)
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def ensure_resource_matches!(rollup)
|
|
84
|
+
return if rollup.resource_type == snapshot.resource_type && rollup.resource_id == snapshot.resource_id
|
|
85
|
+
|
|
86
|
+
raise ResourceConflict,
|
|
87
|
+
"hostname and path are already attributed to another resource for this date"
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def replace_breakdowns(rollup)
|
|
91
|
+
relation = rollup.traffic_breakdowns
|
|
92
|
+
relation.delete_all if snapshot.breakdowns.empty?
|
|
93
|
+
return if snapshot.breakdowns.empty?
|
|
94
|
+
|
|
95
|
+
now = Time.current
|
|
96
|
+
rows = snapshot.breakdowns.map do |breakdown|
|
|
97
|
+
{
|
|
98
|
+
account_id: account.id,
|
|
99
|
+
traffic_rollup_id: rollup.id,
|
|
100
|
+
**breakdown,
|
|
101
|
+
created_at: now,
|
|
102
|
+
updated_at: now
|
|
103
|
+
}
|
|
104
|
+
end
|
|
105
|
+
TrafficBreakdown.upsert_all(
|
|
106
|
+
rows,
|
|
107
|
+
unique_by: BREAKDOWN_INDEX,
|
|
108
|
+
update_only: %i[page_views visits updated_at]
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
kept_ids = snapshot.breakdowns.map do |breakdown|
|
|
112
|
+
relation.find_by!(
|
|
113
|
+
dimension: breakdown[:dimension],
|
|
114
|
+
value: breakdown[:value]
|
|
115
|
+
).id
|
|
116
|
+
end
|
|
117
|
+
relation.where.not(id: kept_ids).delete_all
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
end
|