skadi 0.4.0.beta.1 → 0.4.0.beta.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +4 -0
- data/app/assets/builds/dashboard.css +1 -1
- data/app/assets/builds/dashboard.js +9 -9
- data/app/controllers/skadi/tracking_controller.rb +1 -1
- data/app/models/skadi/dashboard.rb +8 -6
- data/app/models/skadi/dashboard_query.rb +98 -37
- data/app/models/skadi/dashboard_validator.rb +91 -9
- data/app/models/skadi/schema.rb +68 -31
- data/app/models/skadi/visit.rb +1 -1
- data/lib/skadi/configuration.rb +6 -1
- data/lib/skadi/url.rb +14 -6
- data/lib/skadi/version.rb +1 -1
- metadata +2 -2
|
@@ -14,7 +14,7 @@ module Skadi
|
|
|
14
14
|
|
|
15
15
|
def track
|
|
16
16
|
if @params["exit_page"].present? && @params["exit_page"].is_a?(String)
|
|
17
|
-
@view.exit_page = Skadi::Url.redact_and_normalise_url(@params["exit_page"])
|
|
17
|
+
@view.exit_page = Skadi::Url.redact_and_normalise_url(@params["exit_page"], request: request)
|
|
18
18
|
end
|
|
19
19
|
|
|
20
20
|
handle_consent @params["consent"]
|
|
@@ -22,15 +22,17 @@ module Skadi
|
|
|
22
22
|
"title" => "Visits and Checkouts",
|
|
23
23
|
"time_series" => "weekly",
|
|
24
24
|
|
|
25
|
-
"verified_visits" => true,
|
|
26
|
-
"unique_by" => "visit",
|
|
27
|
-
"visit_tracking" => "any",
|
|
28
|
-
|
|
29
25
|
"datasets" => [
|
|
30
26
|
{
|
|
31
27
|
"id" => "7cec5a7a-7bf7-403f-b15e-b2e45944182e",
|
|
32
|
-
"label" => "
|
|
33
|
-
"type" => "
|
|
28
|
+
"label" => "Views",
|
|
29
|
+
"type" => "views",
|
|
30
|
+
"count_by" => "visitors",
|
|
31
|
+
"belongs_to" => {
|
|
32
|
+
"visits" => { "filters" => [
|
|
33
|
+
{ "field" => "verified", "operator" => "=", "value" => true },
|
|
34
|
+
] },
|
|
35
|
+
},
|
|
34
36
|
}.freeze,
|
|
35
37
|
].freeze,
|
|
36
38
|
}.freeze,
|
|
@@ -110,11 +110,12 @@ module Skadi
|
|
|
110
110
|
private def build_query_from_schema(dataset, chart, schema, untrusted_param_filters)
|
|
111
111
|
query = schema[:model].all
|
|
112
112
|
|
|
113
|
+
query = apply_schema_associations(dataset, schema, query)
|
|
113
114
|
query = apply_schema_filters(dataset, schema, query)
|
|
114
115
|
query = apply_chart_filters(dataset, chart, schema, query, untrusted_param_filters)
|
|
115
116
|
|
|
116
117
|
query = if chart["type"] == "table"
|
|
117
|
-
select_table_query(schema, query)
|
|
118
|
+
select_table_query(dataset, schema, query)
|
|
118
119
|
else
|
|
119
120
|
select_split_and_group_chart_query(dataset, chart, schema, query)
|
|
120
121
|
end
|
|
@@ -122,6 +123,35 @@ module Skadi
|
|
|
122
123
|
return query.to_sql
|
|
123
124
|
end
|
|
124
125
|
|
|
126
|
+
private def dataset_each_belongs_to(dataset, schema)
|
|
127
|
+
return unless dataset["belongs_to"].is_a?(Hash)
|
|
128
|
+
|
|
129
|
+
dataset["belongs_to"].each do |belongs_to_table, belongs_to_config|
|
|
130
|
+
next unless belongs_to_config.is_a?(Hash)
|
|
131
|
+
|
|
132
|
+
# Ensure the schema for the current dataset allows joining to the given table
|
|
133
|
+
belongs_to_table = belongs_to_table.to_sym
|
|
134
|
+
next unless schema[:belongs_to]&.key?(belongs_to_table)
|
|
135
|
+
|
|
136
|
+
# Ensure the schema for the joined table exists
|
|
137
|
+
belongs_to_schema = Schema.database_schema[belongs_to_table]
|
|
138
|
+
next unless belongs_to_schema
|
|
139
|
+
|
|
140
|
+
yield(belongs_to_table, belongs_to_config, belongs_to_schema)
|
|
141
|
+
end
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
private def apply_schema_associations(dataset, schema, query)
|
|
145
|
+
dataset_each_belongs_to(dataset, schema) do |belongs_to_table, belongs_to_config, belongs_to_schema|
|
|
146
|
+
join_type = belongs_to_config["required"] ? "INNER JOIN" : "LEFT JOIN"
|
|
147
|
+
model = belongs_to_schema[:model]
|
|
148
|
+
|
|
149
|
+
query = query.joins(%(#{join_type} #{model.table_name} ON #{model.table_name}.id = #{schema[:model].table_name}.#{schema[:belongs_to][belongs_to_table][:key]}))
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
return query
|
|
153
|
+
end
|
|
154
|
+
|
|
125
155
|
POSITIVE_OPERATORS = [ "=", ">", ">=", "<=", "<", "like" ].freeze
|
|
126
156
|
NEGATED_OPERATORS = [ "!=", "not like" ].freeze
|
|
127
157
|
|
|
@@ -186,6 +216,11 @@ module Skadi
|
|
|
186
216
|
end
|
|
187
217
|
end
|
|
188
218
|
|
|
219
|
+
dataset_each_belongs_to(dataset, schema) do |_table, belongs_to_config, belongs_to_schema|
|
|
220
|
+
# Ensure nested belongs_to filters, which aren't supported, don't get applied
|
|
221
|
+
query = apply_schema_filters(belongs_to_config.except("belongs_to"), belongs_to_schema, query)
|
|
222
|
+
end
|
|
223
|
+
|
|
189
224
|
return query
|
|
190
225
|
end
|
|
191
226
|
|
|
@@ -202,42 +237,45 @@ module Skadi
|
|
|
202
237
|
query = query.where("DATE(#{date_config[:sql]}) <= ?", date_to) unless date_to.nil?
|
|
203
238
|
end
|
|
204
239
|
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
if chart["verified_visits"] == true || chart["visit_tracking"] || chart["unique_by"] == "visitor"
|
|
208
|
-
if schema[:model] != Skadi::Visit
|
|
209
|
-
query = query.joins(%(INNER JOIN skadi_visits ON skadi_visits.id = #{schema[:model].table_name}.#{schema[:visit_key]}))
|
|
210
|
-
end
|
|
211
|
-
end
|
|
240
|
+
return query
|
|
241
|
+
end
|
|
212
242
|
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
end
|
|
243
|
+
private def select_table_query(dataset, schema, query)
|
|
244
|
+
return query.select("'No fields selected' as error") unless dataset["selects"].nil? || dataset["selects"].is_a?(Array)
|
|
216
245
|
|
|
217
|
-
|
|
218
|
-
query = query.where("skadi_visits.tracking_token IS NOT NULL")
|
|
219
|
-
elsif chart["visit_tracking"] == "anonymity_set"
|
|
220
|
-
query = query.where("skadi_visits.tracking_token IS NOT NULL AND skadi_visits.cookies_enabled = FALSE")
|
|
221
|
-
elsif chart["visit_tracking"] == "cookie"
|
|
222
|
-
query = query.where("skadi_visits.tracking_token IS NOT NULL AND skadi_visits.cookies_enabled = TRUE")
|
|
223
|
-
end
|
|
224
|
-
end
|
|
246
|
+
selects = dataset["selects"]&.any? ? dataset["selects"] : nil
|
|
225
247
|
|
|
226
|
-
|
|
227
|
-
|
|
248
|
+
# Fall back to selecting all the fields
|
|
249
|
+
selects ||= schema[:fields].keys.map(&:to_s)
|
|
228
250
|
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
251
|
+
select_fields = selects.map do |field|
|
|
252
|
+
table, field = field.split(".", 2) if field.include?(".")
|
|
253
|
+
next unless table.nil? || (schema[:belongs_to]&.key?(table.to_sym) && dataset["belongs_to"].is_a?(Hash) && dataset["belongs_to"][table].is_a?(Hash))
|
|
254
|
+
|
|
255
|
+
field_schema = table.nil? ? schema : Schema.database_schema[table.to_sym]
|
|
256
|
+
next unless field_schema
|
|
257
|
+
|
|
258
|
+
field_config = field_schema[:fields][field.to_sym]
|
|
259
|
+
|
|
260
|
+
next unless field_config && field_config[:select]
|
|
261
|
+
|
|
262
|
+
quoted_field = field_schema[:model].connection.quote_column_name(field)
|
|
263
|
+
quoted_fieldname = if table.nil?
|
|
264
|
+
quoted_field
|
|
265
|
+
else
|
|
266
|
+
field_schema[:model].connection.quote_column_name("#{table}.#{field}")
|
|
267
|
+
end
|
|
233
268
|
|
|
234
269
|
if field_config[:sql]
|
|
235
|
-
"#{field_config[:sql]} AS #{
|
|
270
|
+
"#{field_config[:sql]} AS #{quoted_fieldname}"
|
|
236
271
|
else
|
|
237
|
-
"#{model.table_name}.#{quoted_field}"
|
|
272
|
+
"#{field_schema[:model].table_name}.#{quoted_field} AS #{quoted_fieldname}"
|
|
238
273
|
end
|
|
239
274
|
end
|
|
240
275
|
|
|
276
|
+
select_fields.compact!
|
|
277
|
+
select_fields = [ "'No fields selected' as error" ] unless select_fields.any?
|
|
278
|
+
|
|
241
279
|
return query.select(*select_fields)
|
|
242
280
|
end
|
|
243
281
|
|
|
@@ -248,7 +286,7 @@ module Skadi
|
|
|
248
286
|
safe_id = model.connection.quote(dataset["id"])
|
|
249
287
|
safe_date = "NULL"
|
|
250
288
|
safe_split = "NULL"
|
|
251
|
-
safe_count =
|
|
289
|
+
safe_count = dataset_count(dataset, schema)
|
|
252
290
|
safe_group = split_columns.dup
|
|
253
291
|
|
|
254
292
|
if chart["time_series"].present? && schema[:fields][:date]
|
|
@@ -264,14 +302,6 @@ module Skadi
|
|
|
264
302
|
safe_split = "CONCAT(#{split_columns.join(", '|~|', ")})"
|
|
265
303
|
end
|
|
266
304
|
|
|
267
|
-
if schema[:count_sql]
|
|
268
|
-
safe_count = schema[:count_sql]
|
|
269
|
-
elsif chart["unique_by"] == "visit" && schema[:visit_key]
|
|
270
|
-
safe_count = "COUNT(DISTINCT #{schema[:model].table_name}.#{schema[:visit_key]})"
|
|
271
|
-
elsif chart["unique_by"] == "visitor" && schema[:visit_key]
|
|
272
|
-
safe_count = "COUNT(DISTINCT skadi_visits.tracking_token)"
|
|
273
|
-
end
|
|
274
|
-
|
|
275
305
|
return query
|
|
276
306
|
.select(
|
|
277
307
|
"#{safe_id} AS id",
|
|
@@ -282,12 +312,38 @@ module Skadi
|
|
|
282
312
|
.group(safe_group)
|
|
283
313
|
end
|
|
284
314
|
|
|
315
|
+
private def dataset_count(dataset, schema)
|
|
316
|
+
# Fall back to the name of the current dataset if not set
|
|
317
|
+
count_by = dataset["count_by"].is_a?(String) ? dataset["count_by"].to_sym : dataset["type"].to_sym
|
|
318
|
+
|
|
319
|
+
# Match against the current schema's defined counts, allowing for overriding the default
|
|
320
|
+
return schema[:counts][count_by][:sql] if schema[:counts]&.[](count_by)
|
|
321
|
+
|
|
322
|
+
# Fallback to basic count if counting by the schema table
|
|
323
|
+
return "COUNT(*)" if count_by == dataset["type"].to_sym
|
|
324
|
+
|
|
325
|
+
# If the table is joined, then we can count by using the table's defined counts
|
|
326
|
+
dataset_each_belongs_to(dataset, schema) do |_table, _config, belongs_to_schema|
|
|
327
|
+
return belongs_to_schema[:counts][count_by][:sql] if belongs_to_schema[:counts]&.[](count_by)
|
|
328
|
+
end
|
|
329
|
+
|
|
330
|
+
# Fall back to counting by the foreign key
|
|
331
|
+
if schema[:belongs_to].is_a?(Hash)
|
|
332
|
+
schema[:belongs_to].each do |table, config|
|
|
333
|
+
return "COUNT(DISTINCT #{schema[:model].table_name}.#{config[:key]})" if table == count_by
|
|
334
|
+
end
|
|
335
|
+
end
|
|
336
|
+
|
|
337
|
+
# For invalid configs, we just fallback to a basic count
|
|
338
|
+
return "COUNT(*)"
|
|
339
|
+
end
|
|
340
|
+
|
|
285
341
|
private def safe_split_columns(dataset, schema)
|
|
286
342
|
safe_split = []
|
|
287
343
|
|
|
288
344
|
if dataset["split_by"].is_a?(Array)
|
|
289
345
|
dataset["split_by"].each do |field|
|
|
290
|
-
field_config = schema[:fields][field.to_sym]
|
|
346
|
+
field_config = schema[:fields][field.to_s.to_sym]
|
|
291
347
|
if field_config && field_config[:split]
|
|
292
348
|
safe_field = schema[:model].connection.quote_column_name(field)
|
|
293
349
|
|
|
@@ -296,6 +352,11 @@ module Skadi
|
|
|
296
352
|
end
|
|
297
353
|
end
|
|
298
354
|
|
|
355
|
+
# Now do the same thing for each belongs_to dataset
|
|
356
|
+
dataset_each_belongs_to(dataset, schema) do |_table, belongs_to_config, belongs_to_schema|
|
|
357
|
+
safe_split.append(*safe_split_columns(belongs_to_config.except("belongs_to"), belongs_to_schema))
|
|
358
|
+
end
|
|
359
|
+
|
|
299
360
|
return safe_split
|
|
300
361
|
end
|
|
301
362
|
|
|
@@ -50,9 +50,6 @@ module Skadi
|
|
|
50
50
|
time_series: OneOf.new(allowed_values: %w[daily weekly monthly].freeze, allow_missing: true).freeze,
|
|
51
51
|
date_from: :date?,
|
|
52
52
|
date_to: :date?,
|
|
53
|
-
verified_visits: :boolean?,
|
|
54
|
-
unique_by: OneOf.new(allowed_values: %w[visit visitor].freeze, allow_missing: true).freeze,
|
|
55
|
-
visit_tracking: OneOf.new(allowed_values: %w[any anonymity_set cookie].freeze, allow_missing: true).freeze,
|
|
56
53
|
datasets: ArrayOf.new(:Dataset, 1),
|
|
57
54
|
},
|
|
58
55
|
percentageDataset: {
|
|
@@ -74,10 +71,12 @@ module Skadi
|
|
|
74
71
|
dataset_type = COMMON_DATASET_FIELDS.dup
|
|
75
72
|
dataset_type[:type] = dataset_name.to_s
|
|
76
73
|
|
|
74
|
+
select_fields = []
|
|
77
75
|
split_by_fields = []
|
|
78
76
|
filters = {}
|
|
79
77
|
|
|
80
78
|
schema[:fields].each do |field, field_config|
|
|
79
|
+
select_fields << field.to_s if field_config[:select]
|
|
81
80
|
split_by_fields << field.to_s if field_config[:split]
|
|
82
81
|
next unless field_config[:filter]
|
|
83
82
|
|
|
@@ -88,12 +87,53 @@ module Skadi
|
|
|
88
87
|
end
|
|
89
88
|
end
|
|
90
89
|
|
|
90
|
+
schema[:belongs_to]&.keys&.each do |belongs_to_table|
|
|
91
|
+
Schema.database_schema[belongs_to_table][:fields].each do |field, field_config|
|
|
92
|
+
select_fields << "#{belongs_to_table}.#{field}" if field_config[:select]
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
dataset_type[:count_by] = OneOf.new(allowed_values: [
|
|
97
|
+
dataset_name.to_s,
|
|
98
|
+
# This may result in duplication if the default is overridden, but it will still function correctly
|
|
99
|
+
*schema[:counts]&.keys&.map(&:to_s),
|
|
100
|
+
*schema[:belongs_to]&.keys&.flat_map do |belongs_to_table|
|
|
101
|
+
[
|
|
102
|
+
belongs_to_table.to_s,
|
|
103
|
+
*Schema.database_schema[belongs_to_table][:counts]&.keys&.map(&:to_s),
|
|
104
|
+
]
|
|
105
|
+
end,
|
|
106
|
+
nil,
|
|
107
|
+
], allow_missing: true)
|
|
108
|
+
|
|
109
|
+
dataset_type[:selects] = ArrayOf.new(OneOf.new(select_fields, false)) if select_fields.any?
|
|
91
110
|
dataset_type[:split_by] = ArrayOf.new(OneOf.new(split_by_fields, false)) if split_by_fields.any?
|
|
92
|
-
dataset_type[:filters] = ArrayOf.new(Filter.new(filters))
|
|
111
|
+
dataset_type[:filters] = ArrayOf.new(Filter.new(filters)) if filters.any?
|
|
93
112
|
|
|
94
113
|
@types[:"#{dataset_name}Dataset"] = dataset_type
|
|
95
114
|
end
|
|
96
115
|
|
|
116
|
+
# Second pass to add the validations for belongs_to fields
|
|
117
|
+
Schema.database_schema.each do |dataset_name, schema|
|
|
118
|
+
next unless schema[:belongs_to].is_a?(Hash) && schema[:belongs_to].any?
|
|
119
|
+
|
|
120
|
+
dataset_type = @types[:"#{dataset_name}Dataset"]
|
|
121
|
+
|
|
122
|
+
dataset_type[:belongs_to?] = {}
|
|
123
|
+
schema[:belongs_to].keys.each do |association|
|
|
124
|
+
target_table = @types[:"#{association}Dataset"]
|
|
125
|
+
|
|
126
|
+
# Double check that this association is actually in the schema
|
|
127
|
+
next unless target_table.is_a?(Hash)
|
|
128
|
+
|
|
129
|
+
association_type = { required: :boolean? }
|
|
130
|
+
association_type[:split_by] = target_table[:split_by] if target_table[:split_by]
|
|
131
|
+
association_type[:filters] = target_table[:filters] if target_table[:filters]
|
|
132
|
+
|
|
133
|
+
dataset_type[:belongs_to?][:"#{association}?"] = association_type
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
|
|
97
137
|
return @types
|
|
98
138
|
end
|
|
99
139
|
|
|
@@ -163,13 +203,21 @@ module Skadi
|
|
|
163
203
|
private def validate_hash(type, value, path, context:)
|
|
164
204
|
return add_error(path, "must be a hash", context:) unless value.is_a?(Hash)
|
|
165
205
|
|
|
166
|
-
|
|
206
|
+
valid_keys = type.keys.map do |key|
|
|
207
|
+
key.to_s.delete_suffix(??)
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
extra_keys = value.keys.map(&:to_s) - valid_keys
|
|
167
211
|
extra_keys.each do |key|
|
|
168
212
|
add_error("#{path}.#{key}", "is not a valid key", context:)
|
|
169
213
|
end
|
|
170
214
|
|
|
171
215
|
type.each do |item_key, item_type|
|
|
172
|
-
|
|
216
|
+
optional = item_key.end_with?(??)
|
|
217
|
+
key = item_key.to_s.delete_suffix(??)
|
|
218
|
+
|
|
219
|
+
next if optional && value[key].nil?
|
|
220
|
+
validate_type(item_type, value[key], "#{path}.#{key}", context:)
|
|
173
221
|
end
|
|
174
222
|
end
|
|
175
223
|
|
|
@@ -222,10 +270,44 @@ module Skadi
|
|
|
222
270
|
private def validate_dataset(value, path, context:)
|
|
223
271
|
return add_error(path, "must be a hash", context:) unless value.is_a?(Hash)
|
|
224
272
|
|
|
225
|
-
|
|
226
|
-
return add_error("#{path}.type", "is not a valid dataset type", context:) unless self.class.types.include?(
|
|
273
|
+
type_name = :"#{value["type"]}Dataset"
|
|
274
|
+
return add_error("#{path}.type", "is not a valid dataset type", context:) unless self.class.types.include?(type_name)
|
|
275
|
+
|
|
276
|
+
validate_type(self.class.types[type_name], value, path, context:)
|
|
277
|
+
|
|
278
|
+
# Perform additional validation of the select fields to ensure the right table is included
|
|
279
|
+
|
|
280
|
+
if value["selects"].is_a?(Array)
|
|
281
|
+
value["selects"].each_with_index do |field, index|
|
|
282
|
+
next unless field.is_a?(String) && field.include?(".")
|
|
283
|
+
table, _ = field.split(".", 2)
|
|
284
|
+
|
|
285
|
+
# Ensure that the relevant table is joined
|
|
286
|
+
if !value["belongs_to"]&.key?(table)
|
|
287
|
+
add_error("#{path}.selects[#{index}]", "#{field.inspect} requires a join with the #{table} table", context:)
|
|
288
|
+
end
|
|
289
|
+
end
|
|
290
|
+
end
|
|
291
|
+
|
|
292
|
+
# Perform additional validation of the count_by to ensure the right table is included
|
|
293
|
+
# Early return if the count is the default or is not a string
|
|
294
|
+
return if !value["count_by"].is_a?(String) || value["count_by"] == value["type"]
|
|
295
|
+
|
|
296
|
+
count_by = value["count_by"].to_sym
|
|
297
|
+
|
|
298
|
+
# Early return if the count is defined or is a foreign key on the current dataset
|
|
299
|
+
schema = Schema.database_schema[value["type"].to_sym]
|
|
300
|
+
return if schema[:counts]&.key?(count_by) || schema[:belongs_to]&.key?(count_by)
|
|
227
301
|
|
|
228
|
-
|
|
302
|
+
# Find the table that has the given count as one of its counts
|
|
303
|
+
owner = schema[:belongs_to]&.keys&.find do |table|
|
|
304
|
+
Schema.database_schema[table][:counts]&.key?(count_by)
|
|
305
|
+
end
|
|
306
|
+
|
|
307
|
+
# Ensure that the relevant table is joined
|
|
308
|
+
if owner && !value["belongs_to"]&.key?(owner.to_s)
|
|
309
|
+
add_error("#{path}.count_by", "#{value["count_by"].inspect} requires a join with the #{owner} table", context:)
|
|
310
|
+
end
|
|
229
311
|
end
|
|
230
312
|
|
|
231
313
|
private def validate_custom_type(type, value, path, context:)
|
data/app/models/skadi/schema.rb
CHANGED
|
@@ -2,8 +2,8 @@ module Skadi
|
|
|
2
2
|
class Schema
|
|
3
3
|
class << self
|
|
4
4
|
# Extract out the most common field config into a const
|
|
5
|
-
|
|
6
|
-
private_constant :
|
|
5
|
+
SELECT_FILTER_AND_SPLIT = { select: true, filter: true, split: true }
|
|
6
|
+
private_constant :SELECT_FILTER_AND_SPLIT
|
|
7
7
|
|
|
8
8
|
# A hash where the key represents a dataset name and the value is its configuration
|
|
9
9
|
def database_schema
|
|
@@ -12,99 +12,133 @@ module Skadi
|
|
|
12
12
|
@_database_schema ||= {
|
|
13
13
|
# A hash with the following keys:
|
|
14
14
|
# model: The model for this dataset
|
|
15
|
-
#
|
|
16
|
-
#
|
|
15
|
+
# counts: A hash describing how this table can be counted, e.g. visitors: { sql: "COUNT(DISTINCT skadi_visits.tracking_token)" }
|
|
16
|
+
# If the name of the count matches the table, the specified count overwrites the default of `COUNT(*)`
|
|
17
|
+
# belongs_to: A hash describing belongs_to relationships, e.g. {visits: {key: :visit_id}}
|
|
17
18
|
# fields: A hash where the key represents a field and the value is its configuration
|
|
18
19
|
visits: {
|
|
19
20
|
model: Skadi::Visit,
|
|
20
|
-
|
|
21
|
+
counts: {
|
|
22
|
+
visitors: { sql: "COUNT(DISTINCT skadi_visits.tracking_token)" },
|
|
23
|
+
},
|
|
21
24
|
# If the table has a date field, the fields Hash expects a value for :date
|
|
22
25
|
fields: {
|
|
23
26
|
# A hash with the following keys:
|
|
24
27
|
# label: The label to show in the front end
|
|
25
28
|
# type: The datatype, one of :one_of, :date, :string, :number, :boolean. Defaults to :string.
|
|
29
|
+
# select: True if this field can be selected by the table chart
|
|
26
30
|
# filter: True if this field can be filtered
|
|
27
31
|
# split: True if this field can be split
|
|
28
32
|
# sql: The SQL expression used to get this value for derived fields
|
|
29
33
|
# options: A list of possible values
|
|
30
34
|
# description: used in the front end as help text for this field
|
|
35
|
+
id: {
|
|
36
|
+
select: true,
|
|
37
|
+
},
|
|
31
38
|
date: {
|
|
32
39
|
type: :date,
|
|
40
|
+
select: true,
|
|
33
41
|
filter: true,
|
|
34
42
|
sql: "skadi_visits.created_at",
|
|
35
43
|
},
|
|
36
|
-
|
|
44
|
+
verified: {
|
|
45
|
+
**SELECT_FILTER_AND_SPLIT,
|
|
46
|
+
type: :boolean,
|
|
47
|
+
},
|
|
48
|
+
tracked: {
|
|
49
|
+
**SELECT_FILTER_AND_SPLIT,
|
|
50
|
+
type: :boolean,
|
|
51
|
+
sql: "(skadi_visits.tracking_token IS NOT NULL)",
|
|
52
|
+
description: "Whether the visit is tracked across multiple views and events using a cookie or anonymity set.",
|
|
53
|
+
},
|
|
54
|
+
cookies_enabled: {
|
|
55
|
+
**SELECT_FILTER_AND_SPLIT,
|
|
56
|
+
type: :boolean,
|
|
57
|
+
description: "Whether this visit is tracked across multiple days using cookies.",
|
|
58
|
+
},
|
|
59
|
+
landing_page: SELECT_FILTER_AND_SPLIT,
|
|
37
60
|
referrer_domain: {
|
|
38
|
-
**
|
|
61
|
+
**SELECT_FILTER_AND_SPLIT,
|
|
39
62
|
sql: Helpers::Sql.string_before_separator(Skadi::Visit, "referrer", "/"),
|
|
40
63
|
},
|
|
41
|
-
utm_source:
|
|
42
|
-
utm_medium:
|
|
43
|
-
utm_term:
|
|
44
|
-
utm_content:
|
|
45
|
-
utm_campaign:
|
|
64
|
+
utm_source: SELECT_FILTER_AND_SPLIT,
|
|
65
|
+
utm_medium: SELECT_FILTER_AND_SPLIT,
|
|
66
|
+
utm_term: SELECT_FILTER_AND_SPLIT,
|
|
67
|
+
utm_content: SELECT_FILTER_AND_SPLIT,
|
|
68
|
+
utm_campaign: SELECT_FILTER_AND_SPLIT,
|
|
46
69
|
},
|
|
47
70
|
},
|
|
48
71
|
views: {
|
|
49
72
|
model: Skadi::View,
|
|
50
|
-
|
|
73
|
+
belongs_to: { visits: { key: :visit_id } },
|
|
51
74
|
fields: {
|
|
75
|
+
id: {
|
|
76
|
+
select: true,
|
|
77
|
+
},
|
|
52
78
|
date: {
|
|
53
79
|
type: :date,
|
|
80
|
+
select: true,
|
|
54
81
|
filter: true,
|
|
55
82
|
sql: "skadi_views.created_at",
|
|
56
83
|
},
|
|
57
84
|
verified: {
|
|
85
|
+
**SELECT_FILTER_AND_SPLIT,
|
|
58
86
|
type: :boolean,
|
|
59
|
-
filter: true,
|
|
60
87
|
},
|
|
61
|
-
controller:
|
|
88
|
+
controller: SELECT_FILTER_AND_SPLIT,
|
|
62
89
|
action: {
|
|
90
|
+
select: true,
|
|
63
91
|
filter: true,
|
|
64
92
|
},
|
|
65
93
|
controller_and_action: {
|
|
66
94
|
split: true,
|
|
67
95
|
sql: "CONCAT(skadi_views.controller, '::', skadi_views.action)",
|
|
68
96
|
},
|
|
69
|
-
path:
|
|
97
|
+
path: SELECT_FILTER_AND_SPLIT,
|
|
70
98
|
verb: {
|
|
99
|
+
**SELECT_FILTER_AND_SPLIT,
|
|
71
100
|
type: :one_of,
|
|
72
|
-
filter: true,
|
|
73
|
-
split: true,
|
|
74
101
|
description: "Typically, GET requests are page views, and POST, PUT, PATCH and DELETE are form submissions.",
|
|
75
102
|
options: %w[GET POST PUT PATCH DELETE],
|
|
76
103
|
},
|
|
77
|
-
version:
|
|
78
|
-
exit_page:
|
|
104
|
+
version: SELECT_FILTER_AND_SPLIT,
|
|
105
|
+
exit_page: SELECT_FILTER_AND_SPLIT,
|
|
79
106
|
},
|
|
80
107
|
},
|
|
81
108
|
events: {
|
|
82
109
|
model: Skadi::Event,
|
|
83
|
-
|
|
110
|
+
belongs_to: { visits: { key: :visit_id }, views: { key: :view_id } },
|
|
84
111
|
fields: {
|
|
112
|
+
id: {
|
|
113
|
+
select: true,
|
|
114
|
+
},
|
|
85
115
|
date: {
|
|
86
116
|
type: :date,
|
|
117
|
+
select: true,
|
|
87
118
|
filter: true,
|
|
88
119
|
sql: "skadi_events.created_at",
|
|
89
120
|
},
|
|
90
|
-
name:
|
|
121
|
+
name: SELECT_FILTER_AND_SPLIT,
|
|
91
122
|
**(Skadi.configuration.dashboard_custom_event_fields || {}),
|
|
92
123
|
},
|
|
93
124
|
},
|
|
94
125
|
demographics: {
|
|
95
126
|
model: Demographic,
|
|
96
|
-
|
|
127
|
+
counts: {
|
|
128
|
+
demographics: { label: "Count", sql: "COALESCE(SUM(skadi_demographics.count), 0)" },
|
|
129
|
+
},
|
|
97
130
|
fields: {
|
|
98
131
|
date: {
|
|
99
132
|
type: :date,
|
|
133
|
+
select: true,
|
|
100
134
|
filter: true,
|
|
101
135
|
sql: "skadi_demographics.recorded_on",
|
|
102
136
|
},
|
|
103
|
-
uri:
|
|
104
|
-
name:
|
|
105
|
-
value:
|
|
137
|
+
uri: SELECT_FILTER_AND_SPLIT,
|
|
138
|
+
name: SELECT_FILTER_AND_SPLIT,
|
|
139
|
+
value: SELECT_FILTER_AND_SPLIT,
|
|
106
140
|
count: {
|
|
107
|
-
**
|
|
141
|
+
**SELECT_FILTER_AND_SPLIT,
|
|
108
142
|
type: :number,
|
|
109
143
|
},
|
|
110
144
|
},
|
|
@@ -113,19 +147,22 @@ module Skadi
|
|
|
113
147
|
}
|
|
114
148
|
end
|
|
115
149
|
|
|
150
|
+
ALLOWED_FRONTEND_DATASET_KEYS = [ :fields, :belongs_to, :counts ].freeze
|
|
151
|
+
|
|
116
152
|
def frontend_schema
|
|
117
153
|
return @_frontend_schema if defined?(@_frontend_schema)
|
|
118
154
|
|
|
119
155
|
# Redact the SQL from the dashboard schema
|
|
120
156
|
schema = database_schema.deep_dup
|
|
121
157
|
schema.each_value do |dataset|
|
|
158
|
+
dataset[:belongs_to] = dataset[:belongs_to].keys if dataset.key?(:belongs_to)
|
|
159
|
+
|
|
122
160
|
dataset.keys.each do |key|
|
|
123
|
-
dataset.delete(key) unless key
|
|
161
|
+
dataset.delete(key) unless ALLOWED_FRONTEND_DATASET_KEYS.include?(key)
|
|
124
162
|
end
|
|
125
163
|
|
|
126
|
-
dataset[:
|
|
127
|
-
|
|
128
|
-
end
|
|
164
|
+
dataset[:counts]&.each_value { |field| field.delete(:sql) }
|
|
165
|
+
dataset[:fields].each_value { |field| field.delete(:sql) }
|
|
129
166
|
end
|
|
130
167
|
|
|
131
168
|
@_frontend_schema = schema
|
data/app/models/skadi/visit.rb
CHANGED
|
@@ -42,7 +42,7 @@ module Skadi
|
|
|
42
42
|
tracking_token: tracking_token,
|
|
43
43
|
user_id: user_id,
|
|
44
44
|
|
|
45
|
-
referrer: request ? Skadi::Url.redact_and_normalise_url(request.referrer) : nil,
|
|
45
|
+
referrer: request ? Skadi::Url.redact_and_normalise_url(request.referrer, request: request) : nil,
|
|
46
46
|
landing_page: request ? Skadi::Url.view_path_from_request(request) : nil,
|
|
47
47
|
|
|
48
48
|
utm_source: request ? request.query_parameters["utm_source"] : nil,
|
data/lib/skadi/configuration.rb
CHANGED
|
@@ -148,6 +148,7 @@ module Skadi
|
|
|
148
148
|
# {
|
|
149
149
|
# label: The label to show in the front end,
|
|
150
150
|
# type: The datatype, one of :date, :string, :number, :boolean. Defaults to :string if omitted.
|
|
151
|
+
# select: `true` if this field can be selected by the table chart
|
|
151
152
|
# filter: `true` if this field can be filtered
|
|
152
153
|
# split: `true` if this field can be split
|
|
153
154
|
# sql: The SQL expression used to get this value for derived fields
|
|
@@ -163,8 +164,10 @@ module Skadi
|
|
|
163
164
|
next true if it.nil?
|
|
164
165
|
next false unless it.is_a?(Hash)
|
|
165
166
|
|
|
167
|
+
it.deep_symbolize_keys!
|
|
168
|
+
|
|
166
169
|
next it.all? do |_key, item|
|
|
167
|
-
item.is_a?(Hash) && (item.keys.map(&:to_s) - %w[label type filter split sql
|
|
170
|
+
item.is_a?(Hash) && (item.keys.map(&:to_s) - %w[label type select filter split sql options description]).empty?
|
|
168
171
|
end
|
|
169
172
|
end
|
|
170
173
|
|
|
@@ -177,6 +180,8 @@ module Skadi
|
|
|
177
180
|
next false
|
|
178
181
|
end
|
|
179
182
|
|
|
183
|
+
it.deep_symbolize_keys!
|
|
184
|
+
|
|
180
185
|
tables_valid = it.all? do |_table_name, table_schema|
|
|
181
186
|
unless table_schema.is_a?(Hash)
|
|
182
187
|
error! "Skadi.configuration.dashboard_custom_schema values must be a hash"
|
data/lib/skadi/url.rb
CHANGED
|
@@ -33,8 +33,9 @@ module Skadi
|
|
|
33
33
|
|
|
34
34
|
# Strips non-whitelisted query params and normalises URLs
|
|
35
35
|
# @param url [String]
|
|
36
|
+
# @param request [ActionDispatch::Request, nil] the current request - used to truncate the current host from URLs
|
|
36
37
|
# @return [String, nil]
|
|
37
|
-
def self.redact_and_normalise_url(url)
|
|
38
|
+
def self.redact_and_normalise_url(url, request: nil)
|
|
38
39
|
return nil unless url.is_a?(String) && url.present?
|
|
39
40
|
|
|
40
41
|
uri = URI.parse(url[0, Skadi.configuration.max_url_length])
|
|
@@ -45,16 +46,23 @@ module Skadi
|
|
|
45
46
|
|
|
46
47
|
result = +""
|
|
47
48
|
|
|
49
|
+
interesting_scheme = uri.scheme.present? && ![ "http", "https" ].include?(uri.scheme)
|
|
48
50
|
# Only record interesting schemes, e.g. "android-app://"
|
|
49
|
-
result += "#{uri.scheme}://" if
|
|
51
|
+
result += "#{uri.scheme}://" if interesting_scheme
|
|
50
52
|
|
|
51
|
-
|
|
53
|
+
host_with_port = +""
|
|
54
|
+
if uri.host.present?
|
|
55
|
+
host_with_port << uri.host
|
|
56
|
+
# Only include port if it's non-standard
|
|
57
|
+
host_with_port << ":#{uri.port}" if uri.port != uri.default_port
|
|
58
|
+
end
|
|
52
59
|
|
|
53
|
-
|
|
54
|
-
result <<
|
|
60
|
+
is_local = request.present? && host_with_port == request.host_with_port
|
|
61
|
+
result << host_with_port if interesting_scheme || Skadi.configuration.store_domain_in_views || !is_local
|
|
55
62
|
|
|
63
|
+
path = uri.path&.start_with?("/") ? uri.path : "/#{uri.path}"
|
|
56
64
|
# Normalise the trailing slash
|
|
57
|
-
result << (
|
|
65
|
+
result << (path == "/" ? "/" : path.delete_suffix("/"))
|
|
58
66
|
|
|
59
67
|
result << (param_string.present? ? "?#{param_string}" : "")
|
|
60
68
|
|