skadi 0.4.0.beta.1 → 0.4.0.beta.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -14,7 +14,7 @@ module Skadi
14
14
 
15
15
  def track
16
16
  if @params["exit_page"].present? && @params["exit_page"].is_a?(String)
17
- @view.exit_page = Skadi::Url.redact_and_normalise_url(@params["exit_page"])
17
+ @view.exit_page = Skadi::Url.redact_and_normalise_url(@params["exit_page"], request: request)
18
18
  end
19
19
 
20
20
  handle_consent @params["consent"]
@@ -22,15 +22,17 @@ module Skadi
22
22
  "title" => "Visits and Checkouts",
23
23
  "time_series" => "weekly",
24
24
 
25
- "verified_visits" => true,
26
- "unique_by" => "visit",
27
- "visit_tracking" => "any",
28
-
29
25
  "datasets" => [
30
26
  {
31
27
  "id" => "7cec5a7a-7bf7-403f-b15e-b2e45944182e",
32
- "label" => "Visits",
33
- "type" => "visits",
28
+ "label" => "Views",
29
+ "type" => "views",
30
+ "count_by" => "visitors",
31
+ "belongs_to" => {
32
+ "visits" => { "filters" => [
33
+ { "field" => "verified", "operator" => "=", "value" => true },
34
+ ] },
35
+ },
34
36
  }.freeze,
35
37
  ].freeze,
36
38
  }.freeze,
@@ -110,11 +110,12 @@ module Skadi
110
110
  private def build_query_from_schema(dataset, chart, schema, untrusted_param_filters)
111
111
  query = schema[:model].all
112
112
 
113
+ query = apply_schema_associations(dataset, schema, query)
113
114
  query = apply_schema_filters(dataset, schema, query)
114
115
  query = apply_chart_filters(dataset, chart, schema, query, untrusted_param_filters)
115
116
 
116
117
  query = if chart["type"] == "table"
117
- select_table_query(schema, query)
118
+ select_table_query(dataset, schema, query)
118
119
  else
119
120
  select_split_and_group_chart_query(dataset, chart, schema, query)
120
121
  end
@@ -122,6 +123,35 @@ module Skadi
122
123
  return query.to_sql
123
124
  end
124
125
 
126
+ private def dataset_each_belongs_to(dataset, schema)
127
+ return unless dataset["belongs_to"].is_a?(Hash)
128
+
129
+ dataset["belongs_to"].each do |belongs_to_table, belongs_to_config|
130
+ next unless belongs_to_config.is_a?(Hash)
131
+
132
+ # Ensure the schema for the current dataset allows joining to the given table
133
+ belongs_to_table = belongs_to_table.to_sym
134
+ next unless schema[:belongs_to]&.key?(belongs_to_table)
135
+
136
+ # Ensure the schema for the joined table exists
137
+ belongs_to_schema = Schema.database_schema[belongs_to_table]
138
+ next unless belongs_to_schema
139
+
140
+ yield(belongs_to_table, belongs_to_config, belongs_to_schema)
141
+ end
142
+ end
143
+
144
+ private def apply_schema_associations(dataset, schema, query)
145
+ dataset_each_belongs_to(dataset, schema) do |belongs_to_table, belongs_to_config, belongs_to_schema|
146
+ join_type = belongs_to_config["required"] ? "INNER JOIN" : "LEFT JOIN"
147
+ model = belongs_to_schema[:model]
148
+
149
+ query = query.joins(%(#{join_type} #{model.table_name} ON #{model.table_name}.id = #{schema[:model].table_name}.#{schema[:belongs_to][belongs_to_table][:key]}))
150
+ end
151
+
152
+ return query
153
+ end
154
+
125
155
  POSITIVE_OPERATORS = [ "=", ">", ">=", "<=", "<", "like" ].freeze
126
156
  NEGATED_OPERATORS = [ "!=", "not like" ].freeze
127
157
 
@@ -186,6 +216,11 @@ module Skadi
186
216
  end
187
217
  end
188
218
 
219
+ dataset_each_belongs_to(dataset, schema) do |_table, belongs_to_config, belongs_to_schema|
220
+ # Ensure nested belongs_to filters, which aren't supported, don't get applied
221
+ query = apply_schema_filters(belongs_to_config.except("belongs_to"), belongs_to_schema, query)
222
+ end
223
+
189
224
  return query
190
225
  end
191
226
 
@@ -202,42 +237,45 @@ module Skadi
202
237
  query = query.where("DATE(#{date_config[:sql]}) <= ?", date_to) unless date_to.nil?
203
238
  end
204
239
 
205
- if schema[:visit_key]
206
- # Join the visits if we need to
207
- if chart["verified_visits"] == true || chart["visit_tracking"] || chart["unique_by"] == "visitor"
208
- if schema[:model] != Skadi::Visit
209
- query = query.joins(%(INNER JOIN skadi_visits ON skadi_visits.id = #{schema[:model].table_name}.#{schema[:visit_key]}))
210
- end
211
- end
240
+ return query
241
+ end
212
242
 
213
- if chart["verified_visits"] == true
214
- query = query.where("skadi_visits.verified = TRUE")
215
- end
243
+ private def select_table_query(dataset, schema, query)
244
+ return query.select("'No fields selected' as error") unless dataset["selects"].nil? || dataset["selects"].is_a?(Array)
216
245
 
217
- if chart["visit_tracking"] == "any"
218
- query = query.where("skadi_visits.tracking_token IS NOT NULL")
219
- elsif chart["visit_tracking"] == "anonymity_set"
220
- query = query.where("skadi_visits.tracking_token IS NOT NULL AND skadi_visits.cookies_enabled = FALSE")
221
- elsif chart["visit_tracking"] == "cookie"
222
- query = query.where("skadi_visits.tracking_token IS NOT NULL AND skadi_visits.cookies_enabled = TRUE")
223
- end
224
- end
246
+ selects = dataset["selects"]&.any? ? dataset["selects"] : nil
225
247
 
226
- return query
227
- end
248
+ # Fall back to selecting all the fields
249
+ selects ||= schema[:fields].keys.map(&:to_s)
228
250
 
229
- private def select_table_query(schema, query)
230
- model = schema[:model]
231
- select_fields = schema[:fields].map do |key, field_config|
232
- quoted_field = model.connection.quote_column_name(key)
251
+ select_fields = selects.map do |field|
252
+ table, field = field.split(".", 2) if field.include?(".")
253
+ next unless table.nil? || (schema[:belongs_to]&.key?(table.to_sym) && dataset["belongs_to"].is_a?(Hash) && dataset["belongs_to"][table].is_a?(Hash))
254
+
255
+ field_schema = table.nil? ? schema : Schema.database_schema[table.to_sym]
256
+ next unless field_schema
257
+
258
+ field_config = field_schema[:fields][field.to_sym]
259
+
260
+ next unless field_config && field_config[:select]
261
+
262
+ quoted_field = field_schema[:model].connection.quote_column_name(field)
263
+ quoted_fieldname = if table.nil?
264
+ quoted_field
265
+ else
266
+ field_schema[:model].connection.quote_column_name("#{table}.#{field}")
267
+ end
233
268
 
234
269
  if field_config[:sql]
235
- "#{field_config[:sql]} AS #{quoted_field}"
270
+ "#{field_config[:sql]} AS #{quoted_fieldname}"
236
271
  else
237
- "#{model.table_name}.#{quoted_field}"
272
+ "#{field_schema[:model].table_name}.#{quoted_field} AS #{quoted_fieldname}"
238
273
  end
239
274
  end
240
275
 
276
+ select_fields.compact!
277
+ select_fields = [ "'No fields selected' as error" ] unless select_fields.any?
278
+
241
279
  return query.select(*select_fields)
242
280
  end
243
281
 
@@ -248,7 +286,7 @@ module Skadi
248
286
  safe_id = model.connection.quote(dataset["id"])
249
287
  safe_date = "NULL"
250
288
  safe_split = "NULL"
251
- safe_count = "COUNT(*)"
289
+ safe_count = dataset_count(dataset, schema)
252
290
  safe_group = split_columns.dup
253
291
 
254
292
  if chart["time_series"].present? && schema[:fields][:date]
@@ -264,14 +302,6 @@ module Skadi
264
302
  safe_split = "CONCAT(#{split_columns.join(", '|~|', ")})"
265
303
  end
266
304
 
267
- if schema[:count_sql]
268
- safe_count = schema[:count_sql]
269
- elsif chart["unique_by"] == "visit" && schema[:visit_key]
270
- safe_count = "COUNT(DISTINCT #{schema[:model].table_name}.#{schema[:visit_key]})"
271
- elsif chart["unique_by"] == "visitor" && schema[:visit_key]
272
- safe_count = "COUNT(DISTINCT skadi_visits.tracking_token)"
273
- end
274
-
275
305
  return query
276
306
  .select(
277
307
  "#{safe_id} AS id",
@@ -282,12 +312,38 @@ module Skadi
282
312
  .group(safe_group)
283
313
  end
284
314
 
315
+ private def dataset_count(dataset, schema)
316
+ # Fall back to the name of the current dataset if not set
317
+ count_by = dataset["count_by"].is_a?(String) ? dataset["count_by"].to_sym : dataset["type"].to_sym
318
+
319
+ # Match against the current schema's defined counts, allowing for overriding the default
320
+ return schema[:counts][count_by][:sql] if schema[:counts]&.[](count_by)
321
+
322
+ # Fallback to basic count if counting by the schema table
323
+ return "COUNT(*)" if count_by == dataset["type"].to_sym
324
+
325
+ # If the table is joined, then we can count by using the table's defined counts
326
+ dataset_each_belongs_to(dataset, schema) do |_table, _config, belongs_to_schema|
327
+ return belongs_to_schema[:counts][count_by][:sql] if belongs_to_schema[:counts]&.[](count_by)
328
+ end
329
+
330
+ # Fall back to counting by the foreign key
331
+ if schema[:belongs_to].is_a?(Hash)
332
+ schema[:belongs_to].each do |table, config|
333
+ return "COUNT(DISTINCT #{schema[:model].table_name}.#{config[:key]})" if table == count_by
334
+ end
335
+ end
336
+
337
+ # For invalid configs, we just fallback to a basic count
338
+ return "COUNT(*)"
339
+ end
340
+
285
341
  private def safe_split_columns(dataset, schema)
286
342
  safe_split = []
287
343
 
288
344
  if dataset["split_by"].is_a?(Array)
289
345
  dataset["split_by"].each do |field|
290
- field_config = schema[:fields][field.to_sym]
346
+ field_config = schema[:fields][field.to_s.to_sym]
291
347
  if field_config && field_config[:split]
292
348
  safe_field = schema[:model].connection.quote_column_name(field)
293
349
 
@@ -296,6 +352,11 @@ module Skadi
296
352
  end
297
353
  end
298
354
 
355
+ # Now do the same thing for each belongs_to dataset
356
+ dataset_each_belongs_to(dataset, schema) do |_table, belongs_to_config, belongs_to_schema|
357
+ safe_split.append(*safe_split_columns(belongs_to_config.except("belongs_to"), belongs_to_schema))
358
+ end
359
+
299
360
  return safe_split
300
361
  end
301
362
 
@@ -50,9 +50,6 @@ module Skadi
50
50
  time_series: OneOf.new(allowed_values: %w[daily weekly monthly].freeze, allow_missing: true).freeze,
51
51
  date_from: :date?,
52
52
  date_to: :date?,
53
- verified_visits: :boolean?,
54
- unique_by: OneOf.new(allowed_values: %w[visit visitor].freeze, allow_missing: true).freeze,
55
- visit_tracking: OneOf.new(allowed_values: %w[any anonymity_set cookie].freeze, allow_missing: true).freeze,
56
53
  datasets: ArrayOf.new(:Dataset, 1),
57
54
  },
58
55
  percentageDataset: {
@@ -74,10 +71,12 @@ module Skadi
74
71
  dataset_type = COMMON_DATASET_FIELDS.dup
75
72
  dataset_type[:type] = dataset_name.to_s
76
73
 
74
+ select_fields = []
77
75
  split_by_fields = []
78
76
  filters = {}
79
77
 
80
78
  schema[:fields].each do |field, field_config|
79
+ select_fields << field.to_s if field_config[:select]
81
80
  split_by_fields << field.to_s if field_config[:split]
82
81
  next unless field_config[:filter]
83
82
 
@@ -88,12 +87,53 @@ module Skadi
88
87
  end
89
88
  end
90
89
 
90
+ schema[:belongs_to]&.keys&.each do |belongs_to_table|
91
+ Schema.database_schema[belongs_to_table][:fields].each do |field, field_config|
92
+ select_fields << "#{belongs_to_table}.#{field}" if field_config[:select]
93
+ end
94
+ end
95
+
96
+ dataset_type[:count_by] = OneOf.new(allowed_values: [
97
+ dataset_name.to_s,
98
+ # This may result in duplication if the default is overridden, but it will still function correctly
99
+ *schema[:counts]&.keys&.map(&:to_s),
100
+ *schema[:belongs_to]&.keys&.flat_map do |belongs_to_table|
101
+ [
102
+ belongs_to_table.to_s,
103
+ *Schema.database_schema[belongs_to_table][:counts]&.keys&.map(&:to_s),
104
+ ]
105
+ end,
106
+ nil,
107
+ ], allow_missing: true)
108
+
109
+ dataset_type[:selects] = ArrayOf.new(OneOf.new(select_fields, false)) if select_fields.any?
91
110
  dataset_type[:split_by] = ArrayOf.new(OneOf.new(split_by_fields, false)) if split_by_fields.any?
92
- dataset_type[:filters] = ArrayOf.new(Filter.new(filters))
111
+ dataset_type[:filters] = ArrayOf.new(Filter.new(filters)) if filters.any?
93
112
 
94
113
  @types[:"#{dataset_name}Dataset"] = dataset_type
95
114
  end
96
115
 
116
+ # Second pass to add the validations for belongs_to fields
117
+ Schema.database_schema.each do |dataset_name, schema|
118
+ next unless schema[:belongs_to].is_a?(Hash) && schema[:belongs_to].any?
119
+
120
+ dataset_type = @types[:"#{dataset_name}Dataset"]
121
+
122
+ dataset_type[:belongs_to?] = {}
123
+ schema[:belongs_to].keys.each do |association|
124
+ target_table = @types[:"#{association}Dataset"]
125
+
126
+ # Double check that this association is actually in the schema
127
+ next unless target_table.is_a?(Hash)
128
+
129
+ association_type = { required: :boolean? }
130
+ association_type[:split_by] = target_table[:split_by] if target_table[:split_by]
131
+ association_type[:filters] = target_table[:filters] if target_table[:filters]
132
+
133
+ dataset_type[:belongs_to?][:"#{association}?"] = association_type
134
+ end
135
+ end
136
+
97
137
  return @types
98
138
  end
99
139
 
@@ -163,13 +203,21 @@ module Skadi
163
203
  private def validate_hash(type, value, path, context:)
164
204
  return add_error(path, "must be a hash", context:) unless value.is_a?(Hash)
165
205
 
166
- extra_keys = value.keys.map(&:to_sym) - type.keys
206
+ valid_keys = type.keys.map do |key|
207
+ key.to_s.delete_suffix(??)
208
+ end
209
+
210
+ extra_keys = value.keys.map(&:to_s) - valid_keys
167
211
  extra_keys.each do |key|
168
212
  add_error("#{path}.#{key}", "is not a valid key", context:)
169
213
  end
170
214
 
171
215
  type.each do |item_key, item_type|
172
- validate_type(item_type, value[item_key.to_s], "#{path}.#{item_key}", context:)
216
+ optional = item_key.end_with?(??)
217
+ key = item_key.to_s.delete_suffix(??)
218
+
219
+ next if optional && value[key].nil?
220
+ validate_type(item_type, value[key], "#{path}.#{key}", context:)
173
221
  end
174
222
  end
175
223
 
@@ -222,10 +270,44 @@ module Skadi
222
270
  private def validate_dataset(value, path, context:)
223
271
  return add_error(path, "must be a hash", context:) unless value.is_a?(Hash)
224
272
 
225
- type = :"#{value["type"]}Dataset"
226
- return add_error("#{path}.type", "is not a valid dataset type", context:) unless self.class.types.include?(type)
273
+ type_name = :"#{value["type"]}Dataset"
274
+ return add_error("#{path}.type", "is not a valid dataset type", context:) unless self.class.types.include?(type_name)
275
+
276
+ validate_type(self.class.types[type_name], value, path, context:)
277
+
278
+ # Perform additional validation of the select fields to ensure the right table is included
279
+
280
+ if value["selects"].is_a?(Array)
281
+ value["selects"].each_with_index do |field, index|
282
+ next unless field.is_a?(String) && field.include?(".")
283
+ table, _ = field.split(".", 2)
284
+
285
+ # Ensure that the relevant table is joined
286
+ if !value["belongs_to"]&.key?(table)
287
+ add_error("#{path}.selects[#{index}]", "#{field.inspect} requires a join with the #{table} table", context:)
288
+ end
289
+ end
290
+ end
291
+
292
+ # Perform additional validation of the count_by to ensure the right table is included
293
+ # Early return if the count is the default or is not a string
294
+ return if !value["count_by"].is_a?(String) || value["count_by"] == value["type"]
295
+
296
+ count_by = value["count_by"].to_sym
297
+
298
+ # Early return if the count is defined or is a foreign key on the current dataset
299
+ schema = Schema.database_schema[value["type"].to_sym]
300
+ return if schema[:counts]&.key?(count_by) || schema[:belongs_to]&.key?(count_by)
227
301
 
228
- validate_type(type, value, path, context:)
302
+ # Find the table that has the given count as one of its counts
303
+ owner = schema[:belongs_to]&.keys&.find do |table|
304
+ Schema.database_schema[table][:counts]&.key?(count_by)
305
+ end
306
+
307
+ # Ensure that the relevant table is joined
308
+ if owner && !value["belongs_to"]&.key?(owner.to_s)
309
+ add_error("#{path}.count_by", "#{value["count_by"].inspect} requires a join with the #{owner} table", context:)
310
+ end
229
311
  end
230
312
 
231
313
  private def validate_custom_type(type, value, path, context:)
@@ -2,8 +2,8 @@ module Skadi
2
2
  class Schema
3
3
  class << self
4
4
  # Extract out the most common field config into a const
5
- FILTER_AND_SPLIT = { filter: true, split: true }
6
- private_constant :FILTER_AND_SPLIT
5
+ SELECT_FILTER_AND_SPLIT = { select: true, filter: true, split: true }
6
+ private_constant :SELECT_FILTER_AND_SPLIT
7
7
 
8
8
  # A hash where the key represents a dataset name and the value is its configuration
9
9
  def database_schema
@@ -12,99 +12,133 @@ module Skadi
12
12
  @_database_schema ||= {
13
13
  # A hash with the following keys:
14
14
  # model: The model for this dataset
15
- # visit_key: used for chart-level visit filters
16
- # count_sql: The SQL expression used to count this field. Defaults to COUNT().
15
+ # counts: A hash describing how this table can be counted, e.g. visitors: { sql: "COUNT(DISTINCT skadi_visits.tracking_token)" }
16
+ # If the name of the count matches the table, the specified count overwrites the default of `COUNT(*)`
17
+ # belongs_to: A hash describing belongs_to relationships, e.g. {visits: {key: :visit_id}}
17
18
  # fields: A hash where the key represents a field and the value is its configuration
18
19
  visits: {
19
20
  model: Skadi::Visit,
20
- visit_key: "id",
21
+ counts: {
22
+ visitors: { sql: "COUNT(DISTINCT skadi_visits.tracking_token)" },
23
+ },
21
24
  # If the table has a date field, the fields Hash expects a value for :date
22
25
  fields: {
23
26
  # A hash with the following keys:
24
27
  # label: The label to show in the front end
25
28
  # type: The datatype, one of :one_of, :date, :string, :number, :boolean. Defaults to :string.
29
+ # select: True if this field can be selected by the table chart
26
30
  # filter: True if this field can be filtered
27
31
  # split: True if this field can be split
28
32
  # sql: The SQL expression used to get this value for derived fields
29
33
  # options: A list of possible values
30
34
  # description: used in the front end as help text for this field
35
+ id: {
36
+ select: true,
37
+ },
31
38
  date: {
32
39
  type: :date,
40
+ select: true,
33
41
  filter: true,
34
42
  sql: "skadi_visits.created_at",
35
43
  },
36
- landing_page: FILTER_AND_SPLIT,
44
+ verified: {
45
+ **SELECT_FILTER_AND_SPLIT,
46
+ type: :boolean,
47
+ },
48
+ tracked: {
49
+ **SELECT_FILTER_AND_SPLIT,
50
+ type: :boolean,
51
+ sql: "(skadi_visits.tracking_token IS NOT NULL)",
52
+ description: "Whether the visit is tracked across multiple views and events using a cookie or anonymity set.",
53
+ },
54
+ cookies_enabled: {
55
+ **SELECT_FILTER_AND_SPLIT,
56
+ type: :boolean,
57
+ description: "Whether this visit is tracked across multiple days using cookies.",
58
+ },
59
+ landing_page: SELECT_FILTER_AND_SPLIT,
37
60
  referrer_domain: {
38
- **FILTER_AND_SPLIT,
61
+ **SELECT_FILTER_AND_SPLIT,
39
62
  sql: Helpers::Sql.string_before_separator(Skadi::Visit, "referrer", "/"),
40
63
  },
41
- utm_source: FILTER_AND_SPLIT,
42
- utm_medium: FILTER_AND_SPLIT,
43
- utm_term: FILTER_AND_SPLIT,
44
- utm_content: FILTER_AND_SPLIT,
45
- utm_campaign: FILTER_AND_SPLIT,
64
+ utm_source: SELECT_FILTER_AND_SPLIT,
65
+ utm_medium: SELECT_FILTER_AND_SPLIT,
66
+ utm_term: SELECT_FILTER_AND_SPLIT,
67
+ utm_content: SELECT_FILTER_AND_SPLIT,
68
+ utm_campaign: SELECT_FILTER_AND_SPLIT,
46
69
  },
47
70
  },
48
71
  views: {
49
72
  model: Skadi::View,
50
- visit_key: "visit_id",
73
+ belongs_to: { visits: { key: :visit_id } },
51
74
  fields: {
75
+ id: {
76
+ select: true,
77
+ },
52
78
  date: {
53
79
  type: :date,
80
+ select: true,
54
81
  filter: true,
55
82
  sql: "skadi_views.created_at",
56
83
  },
57
84
  verified: {
85
+ **SELECT_FILTER_AND_SPLIT,
58
86
  type: :boolean,
59
- filter: true,
60
87
  },
61
- controller: FILTER_AND_SPLIT,
88
+ controller: SELECT_FILTER_AND_SPLIT,
62
89
  action: {
90
+ select: true,
63
91
  filter: true,
64
92
  },
65
93
  controller_and_action: {
66
94
  split: true,
67
95
  sql: "CONCAT(skadi_views.controller, '::', skadi_views.action)",
68
96
  },
69
- path: FILTER_AND_SPLIT,
97
+ path: SELECT_FILTER_AND_SPLIT,
70
98
  verb: {
99
+ **SELECT_FILTER_AND_SPLIT,
71
100
  type: :one_of,
72
- filter: true,
73
- split: true,
74
101
  description: "Typically, GET requests are page views, and POST, PUT, PATCH and DELETE are form submissions.",
75
102
  options: %w[GET POST PUT PATCH DELETE],
76
103
  },
77
- version: FILTER_AND_SPLIT,
78
- exit_page: FILTER_AND_SPLIT,
104
+ version: SELECT_FILTER_AND_SPLIT,
105
+ exit_page: SELECT_FILTER_AND_SPLIT,
79
106
  },
80
107
  },
81
108
  events: {
82
109
  model: Skadi::Event,
83
- visit_key: "visit_id",
110
+ belongs_to: { visits: { key: :visit_id }, views: { key: :view_id } },
84
111
  fields: {
112
+ id: {
113
+ select: true,
114
+ },
85
115
  date: {
86
116
  type: :date,
117
+ select: true,
87
118
  filter: true,
88
119
  sql: "skadi_events.created_at",
89
120
  },
90
- name: FILTER_AND_SPLIT,
121
+ name: SELECT_FILTER_AND_SPLIT,
91
122
  **(Skadi.configuration.dashboard_custom_event_fields || {}),
92
123
  },
93
124
  },
94
125
  demographics: {
95
126
  model: Demographic,
96
- count_sql: "COALESCE(SUM(skadi_demographics.count), 0)",
127
+ counts: {
128
+ demographics: { label: "Count", sql: "COALESCE(SUM(skadi_demographics.count), 0)" },
129
+ },
97
130
  fields: {
98
131
  date: {
99
132
  type: :date,
133
+ select: true,
100
134
  filter: true,
101
135
  sql: "skadi_demographics.recorded_on",
102
136
  },
103
- uri: FILTER_AND_SPLIT,
104
- name: FILTER_AND_SPLIT,
105
- value: FILTER_AND_SPLIT,
137
+ uri: SELECT_FILTER_AND_SPLIT,
138
+ name: SELECT_FILTER_AND_SPLIT,
139
+ value: SELECT_FILTER_AND_SPLIT,
106
140
  count: {
107
- **FILTER_AND_SPLIT,
141
+ **SELECT_FILTER_AND_SPLIT,
108
142
  type: :number,
109
143
  },
110
144
  },
@@ -113,19 +147,22 @@ module Skadi
113
147
  }
114
148
  end
115
149
 
150
+ ALLOWED_FRONTEND_DATASET_KEYS = [ :fields, :belongs_to, :counts ].freeze
151
+
116
152
  def frontend_schema
117
153
  return @_frontend_schema if defined?(@_frontend_schema)
118
154
 
119
155
  # Redact the SQL from the dashboard schema
120
156
  schema = database_schema.deep_dup
121
157
  schema.each_value do |dataset|
158
+ dataset[:belongs_to] = dataset[:belongs_to].keys if dataset.key?(:belongs_to)
159
+
122
160
  dataset.keys.each do |key|
123
- dataset.delete(key) unless key == :fields
161
+ dataset.delete(key) unless ALLOWED_FRONTEND_DATASET_KEYS.include?(key)
124
162
  end
125
163
 
126
- dataset[:fields].each_value do |field|
127
- field.delete(:sql)
128
- end
164
+ dataset[:counts]&.each_value { |field| field.delete(:sql) }
165
+ dataset[:fields].each_value { |field| field.delete(:sql) }
129
166
  end
130
167
 
131
168
  @_frontend_schema = schema
@@ -42,7 +42,7 @@ module Skadi
42
42
  tracking_token: tracking_token,
43
43
  user_id: user_id,
44
44
 
45
- referrer: request ? Skadi::Url.redact_and_normalise_url(request.referrer) : nil,
45
+ referrer: request ? Skadi::Url.redact_and_normalise_url(request.referrer, request: request) : nil,
46
46
  landing_page: request ? Skadi::Url.view_path_from_request(request) : nil,
47
47
 
48
48
  utm_source: request ? request.query_parameters["utm_source"] : nil,
@@ -148,6 +148,7 @@ module Skadi
148
148
  # {
149
149
  # label: The label to show in the front end,
150
150
  # type: The datatype, one of :date, :string, :number, :boolean. Defaults to :string if omitted.
151
+ # select: `true` if this field can be selected by the table chart
151
152
  # filter: `true` if this field can be filtered
152
153
  # split: `true` if this field can be split
153
154
  # sql: The SQL expression used to get this value for derived fields
@@ -163,8 +164,10 @@ module Skadi
163
164
  next true if it.nil?
164
165
  next false unless it.is_a?(Hash)
165
166
 
167
+ it.deep_symbolize_keys!
168
+
166
169
  next it.all? do |_key, item|
167
- item.is_a?(Hash) && (item.keys.map(&:to_s) - %w[label type filter split sql option description]).empty?
170
+ item.is_a?(Hash) && (item.keys.map(&:to_s) - %w[label type select filter split sql options description]).empty?
168
171
  end
169
172
  end
170
173
 
@@ -177,6 +180,8 @@ module Skadi
177
180
  next false
178
181
  end
179
182
 
183
+ it.deep_symbolize_keys!
184
+
180
185
  tables_valid = it.all? do |_table_name, table_schema|
181
186
  unless table_schema.is_a?(Hash)
182
187
  error! "Skadi.configuration.dashboard_custom_schema values must be a hash"
data/lib/skadi/url.rb CHANGED
@@ -33,8 +33,9 @@ module Skadi
33
33
 
34
34
  # Strips non-whitelisted query params and normalises URLs
35
35
  # @param url [String]
36
+ # @param request [ActionDispatch::Request, nil] the current request - used to truncate the current host from URLs
36
37
  # @return [String, nil]
37
- def self.redact_and_normalise_url(url)
38
+ def self.redact_and_normalise_url(url, request: nil)
38
39
  return nil unless url.is_a?(String) && url.present?
39
40
 
40
41
  uri = URI.parse(url[0, Skadi.configuration.max_url_length])
@@ -45,16 +46,23 @@ module Skadi
45
46
 
46
47
  result = +""
47
48
 
49
+ interesting_scheme = uri.scheme.present? && ![ "http", "https" ].include?(uri.scheme)
48
50
  # Only record interesting schemes, e.g. "android-app://"
49
- result += "#{uri.scheme}://" if uri.scheme.present? && ![ "http", "https" ].include?(uri.scheme)
51
+ result += "#{uri.scheme}://" if interesting_scheme
50
52
 
51
- result += uri.host if uri.host.present?
53
+ host_with_port = +""
54
+ if uri.host.present?
55
+ host_with_port << uri.host
56
+ # Only include port if it's non-standard
57
+ host_with_port << ":#{uri.port}" if uri.port != uri.default_port
58
+ end
52
59
 
53
- # Only include port if it's non-standard
54
- result << ":#{uri.port}" if uri.port != uri.default_port
60
+ is_local = request.present? && host_with_port == request.host_with_port
61
+ result << host_with_port if interesting_scheme || Skadi.configuration.store_domain_in_views || !is_local
55
62
 
63
+ path = uri.path&.start_with?("/") ? uri.path : "/#{uri.path}"
56
64
  # Normalise the trailing slash
57
- result << ((uri.path == "" || uri.path == "/") ? "/" : uri.path.chomp("/")) unless uri.path.nil?
65
+ result << (path == "/" ? "/" : path.delete_suffix("/"))
58
66
 
59
67
  result << (param_string.present? ? "?#{param_string}" : "")
60
68