strata-cli 0.1.15 → 0.1.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +20 -0
- data/lib/strata/cli/agent_mode.rb +23 -1
- data/lib/strata/cli/descriptions/init.txt +4 -0
- data/lib/strata/cli/generators/project.rb +20 -4
- data/lib/strata/cli/generators/templates/AGENTS.md +4 -0
- data/lib/strata/cli/helpers/browser_auth.rb +52 -0
- data/lib/strata/cli/helpers/command_context.rb +1 -5
- data/lib/strata/cli/helpers/datasource_helper.rb +1 -1
- data/lib/strata/cli/helpers/prompts.rb +27 -0
- data/lib/strata/cli/helpers/semantic_audit_checks.rb +507 -0
- data/lib/strata/cli/sub_commands/audit.rb +24 -6
- data/lib/strata/cli/sub_commands/branch.rb +0 -1
- data/lib/strata/cli/sub_commands/datasource.rb +7 -4
- data/lib/strata/cli/sub_commands/deploy.rb +42 -22
- data/lib/strata/cli/sub_commands/project.rb +2 -42
- data/lib/strata/cli/utils/git.rb +31 -9
- data/lib/strata/cli/utils/sql_expression_parser.rb +153 -0
- data/lib/strata/cli/version.rb +1 -1
- metadata +5 -2
|
@@ -0,0 +1,507 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "date"
|
|
4
|
+
require_relative "../utils/sql_expression_parser"
|
|
5
|
+
|
|
6
|
+
module Strata
|
|
7
|
+
module CLI
|
|
8
|
+
# Local mirrors of the server's deploy-time semantic validations, so
|
|
9
|
+
# `strata audit` (and therefore `strata deploy`, which runs it first)
|
|
10
|
+
# fails fast with named errors instead of failing mid-deploy on the
|
|
11
|
+
# server. Each check cites the server code it mirrors (paths in the
|
|
12
|
+
# strata repo) — keep them in lockstep.
|
|
13
|
+
#
|
|
14
|
+
# Server-only checks (universe/path integrity, physical table existence
|
|
15
|
+
# in the warehouse, licenses) are deliberately not replicated here.
|
|
16
|
+
module SemanticAuditChecks
|
|
17
|
+
# deployer/handlers/table.rb dispatches on the literal string
|
|
18
|
+
# "dimension"; any other value silently becomes a measure and a
|
|
19
|
+
# missing type crashes the deployer.
|
|
20
|
+
FIELD_TYPES = %w[dimension measure].freeze
|
|
21
|
+
# semantic/field.rb enums — invalid values raise a raw ArgumentError
|
|
22
|
+
# on the server.
|
|
23
|
+
DATA_TYPES = %w[string integer decimal date date_time boolean bigint binary].freeze
|
|
24
|
+
DISPLAY_TYPES = %w[default html url email phone_number image].freeze
|
|
25
|
+
DATE_GRAINS = %w[raw millisecond second minute hour day month week quarter year].freeze
|
|
26
|
+
NUMERIC_DATA_TYPES = %w[integer decimal bigint].freeze
|
|
27
|
+
# semantic/measure.rb + measure/excludable.rb + measure/includable.rb
|
|
28
|
+
SNAPSHOT_TYPES = %w[beginning ending].freeze
|
|
29
|
+
EXCLUSION_TYPES = %w[exclude exclude_all_except exclude_all].freeze
|
|
30
|
+
EXCLUSION_ENTITY_TYPES = %w[dimension table universe].freeze
|
|
31
|
+
EXCLUSION_FILTER_TYPES = %w[ignore apply only].freeze
|
|
32
|
+
# semantic/partition.rb + semantic/parseable_date.rb
|
|
33
|
+
PARTITION_PREDICATES = %w[between greater_than greater_than_or_equal_to less_than less_than_or_equal_to].freeze
|
|
34
|
+
DYNAMIC_DATE = /(\d{1,3})([hdwmqy])/i
|
|
35
|
+
# semantic/join_def.rb enums — note: no many_to_many.
|
|
36
|
+
JOIN_TYPES = %w[inner left right].freeze
|
|
37
|
+
# semantic/expression.rb:17 — the server accepts any word(...) call.
|
|
38
|
+
MEASURE_SQL_FORMAT = /\A.*\w+\(.*\).*\z/i
|
|
39
|
+
|
|
40
|
+
private
|
|
41
|
+
|
|
42
|
+
# ---- datasources.yml (semantic/datasource.rb) ----
|
|
43
|
+
|
|
44
|
+
def audit_datasource_definitions
|
|
45
|
+
failures = []
|
|
46
|
+
seen_names = {}
|
|
47
|
+
|
|
48
|
+
datasources.each do |key, config|
|
|
49
|
+
unless config.is_a?(Hash)
|
|
50
|
+
failures << {file: "datasources.yml", message: "Datasource '#{key}' must be a mapping"}
|
|
51
|
+
next
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
adapter = config["adapter"].to_s.downcase.strip
|
|
55
|
+
if adapter.empty?
|
|
56
|
+
failures << {file: "datasources.yml", message: "Datasource '#{key}' missing required 'adapter'"}
|
|
57
|
+
elsif !DWH.adapter?(adapter.to_sym)
|
|
58
|
+
failures << {
|
|
59
|
+
file: "datasources.yml",
|
|
60
|
+
message: "Datasource '#{key}' has unknown adapter '#{adapter}' (known: #{DWH.adapters.keys.join(", ")})"
|
|
61
|
+
}
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
name = config["name"].to_s.downcase.strip
|
|
65
|
+
next if name.empty?
|
|
66
|
+
|
|
67
|
+
if seen_names[name]
|
|
68
|
+
failures << {
|
|
69
|
+
file: "datasources.yml",
|
|
70
|
+
message: "Datasource '#{key}' duplicates the name '#{config["name"]}' used by '#{seen_names[name]}'"
|
|
71
|
+
}
|
|
72
|
+
else
|
|
73
|
+
seen_names[name] = key
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
failures
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# ---- fields (semantic/field.rb, semantic/measure.rb) ----
|
|
81
|
+
|
|
82
|
+
def validate_field_semantics(field, file, failures, meta)
|
|
83
|
+
name = field["name"]
|
|
84
|
+
|
|
85
|
+
unless FIELD_TYPES.include?(field["type"].to_s.downcase.strip)
|
|
86
|
+
failures << {
|
|
87
|
+
file: file,
|
|
88
|
+
message: "Field '#{name}' must have type: dimension or measure (the deployer dispatches on this key)"
|
|
89
|
+
}
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
validate_field_enums(name, field, file, failures)
|
|
93
|
+
validate_measure_rules(name, field, file, failures)
|
|
94
|
+
validate_expression_semantics(name, field, file, failures, meta)
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def validate_field_enums(name, field, file, failures)
|
|
98
|
+
data_type = field["data_type"].to_s.strip
|
|
99
|
+
if data_type.empty?
|
|
100
|
+
failures << {file: file, message: "Field '#{name}' missing required 'data_type' (one of: #{DATA_TYPES.join(", ")})"}
|
|
101
|
+
elsif !DATA_TYPES.include?(data_type)
|
|
102
|
+
failures << {
|
|
103
|
+
file: file,
|
|
104
|
+
message: "Field '#{name}' has invalid data_type '#{data_type}' (expected one of: #{DATA_TYPES.join(", ")})"
|
|
105
|
+
}
|
|
106
|
+
end
|
|
107
|
+
|
|
108
|
+
if field.key?("display_type") && !DISPLAY_TYPES.include?(field["display_type"].to_s)
|
|
109
|
+
failures << {
|
|
110
|
+
file: file,
|
|
111
|
+
message: "Field '#{name}' has invalid display_type '#{field["display_type"]}' (expected one of: #{DISPLAY_TYPES.join(", ")})"
|
|
112
|
+
}
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
invalid_grains = Array(field["grains"]).map(&:to_s) - DATE_GRAINS
|
|
116
|
+
if invalid_grains.any?
|
|
117
|
+
failures << {file: file, message: "Field '#{name}' grains contains invalid values: #{invalid_grains.join(", ")}"}
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def validate_measure_rules(name, field, file, failures)
|
|
122
|
+
if field.key?("snapshot") && !SNAPSHOT_TYPES.include?(field["snapshot"].to_s)
|
|
123
|
+
failures << {
|
|
124
|
+
file: file,
|
|
125
|
+
message: "Field '#{name}' has invalid snapshot '#{field["snapshot"]}' (expected beginning or ending)"
|
|
126
|
+
}
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
exclusion_type = field["exclusion_type"]
|
|
130
|
+
if exclusion_type && !EXCLUSION_TYPES.include?(exclusion_type.to_s)
|
|
131
|
+
failures << {
|
|
132
|
+
file: file,
|
|
133
|
+
message: "Field '#{name}' has invalid exclusion_type '#{exclusion_type}' (expected one of: #{EXCLUSION_TYPES.join(", ")})"
|
|
134
|
+
}
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
exclusions = Array(field["exclusions"])
|
|
138
|
+
if exclusion_type.to_s == "exclude_all" && exclusions.any?
|
|
139
|
+
failures << {file: file, message: "Field '#{name}' exclusions should be empty when exclusion_type is exclude_all"}
|
|
140
|
+
end
|
|
141
|
+
|
|
142
|
+
exclusions.each_with_index do |rule, idx|
|
|
143
|
+
unless rule.is_a?(Hash)
|
|
144
|
+
failures << {file: file, message: "Field '#{name}' exclusion at index #{idx} must be a mapping"}
|
|
145
|
+
next
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
unless EXCLUSION_ENTITY_TYPES.include?(rule["type"].to_s)
|
|
149
|
+
failures << {
|
|
150
|
+
file: file,
|
|
151
|
+
message: "Field '#{name}' exclusion type should be one of: #{EXCLUSION_ENTITY_TYPES.join(", ")}"
|
|
152
|
+
}
|
|
153
|
+
end
|
|
154
|
+
unless EXCLUSION_FILTER_TYPES.include?(rule["filter"].to_s)
|
|
155
|
+
failures << {
|
|
156
|
+
file: file,
|
|
157
|
+
message: "Field '#{name}' exclusion filter should be one of: #{EXCLUSION_FILTER_TYPES.join(", ")}"
|
|
158
|
+
}
|
|
159
|
+
end
|
|
160
|
+
if Array(rule["entities"]).empty?
|
|
161
|
+
failures << {file: file, message: "Field '#{name}' exclusion must list at least one entity"}
|
|
162
|
+
end
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
inclusions = field["inclusions"]
|
|
166
|
+
if inclusions.is_a?(Hash)
|
|
167
|
+
unless EXCLUSION_FILTER_TYPES.include?(inclusions["filter"].to_s)
|
|
168
|
+
failures << {
|
|
169
|
+
file: file,
|
|
170
|
+
message: "Field '#{name}' inclusions filter should be one of: #{EXCLUSION_FILTER_TYPES.join(", ")}"
|
|
171
|
+
}
|
|
172
|
+
end
|
|
173
|
+
if inclusions["aggregation"].to_s.strip.empty?
|
|
174
|
+
failures << {file: file, message: "Field '#{name}' inclusions missing 'aggregation'"}
|
|
175
|
+
end
|
|
176
|
+
if Array(inclusions["dimensions"]).empty?
|
|
177
|
+
failures << {file: file, message: "Field '#{name}' inclusions must list at least one dimension"}
|
|
178
|
+
end
|
|
179
|
+
elsif !inclusions.nil?
|
|
180
|
+
failures << {file: file, message: "Field '#{name}' inclusions must be a mapping"}
|
|
181
|
+
end
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
# ---- expressions (semantic/expression.rb, expression/references.rb) ----
|
|
185
|
+
|
|
186
|
+
def validate_expression_semantics(name, field, file, failures, meta)
|
|
187
|
+
sql = expression_sql(field["expression"])
|
|
188
|
+
return unless sql
|
|
189
|
+
|
|
190
|
+
begin
|
|
191
|
+
parsed = Sql::ExpressionParser.new(sql, reserved: meta[:reserved], aggregates: meta[:aggregates])
|
|
192
|
+
rescue => e
|
|
193
|
+
failures << {file: file, message: "Field '#{name}': expression could not be parsed: #{e.message}"}
|
|
194
|
+
return
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
dimension = field["type"].to_s.downcase.strip == "dimension"
|
|
198
|
+
|
|
199
|
+
# The incident this file exists for: expression.rb:14 requires a
|
|
200
|
+
# dimension's SQL to parse to at least one column token.
|
|
201
|
+
if dimension && parsed.columns.empty?
|
|
202
|
+
failures << {
|
|
203
|
+
file: file,
|
|
204
|
+
message: "Field '#{name}': dimensions should reference a table column " \
|
|
205
|
+
"(literals, reserved words and [Field] references alone don't count; quote a column whose name is a reserved word)"
|
|
206
|
+
}
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
# expression.rb:17 — skipped server-side when column_refs is empty
|
|
210
|
+
# (count(*)-style measures).
|
|
211
|
+
if !dimension && parsed.columns.any? && !sql.match?(MEASURE_SQL_FORMAT)
|
|
212
|
+
failures << {file: file, message: "Field '#{name}': measure should have an aggregation function"}
|
|
213
|
+
end
|
|
214
|
+
|
|
215
|
+
validate_reference_resolution(name, dimension, parsed, file, failures)
|
|
216
|
+
end
|
|
217
|
+
|
|
218
|
+
def validate_reference_resolution(name, dimension, parsed, file, failures)
|
|
219
|
+
fields = project_semantic_index[:fields]
|
|
220
|
+
measure_refs = parsed.measure_refs.dup
|
|
221
|
+
|
|
222
|
+
parsed.measure_refs.each do |ref|
|
|
223
|
+
unless (fields[ref] || []).include?("measure")
|
|
224
|
+
failures << {file: file, message: "Field '#{name}': [#{ref}]@m measure was not found in the project"}
|
|
225
|
+
end
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
parsed.dimension_refs.each do |ref|
|
|
229
|
+
unless (fields[ref] || []).include?("dimension")
|
|
230
|
+
failures << {file: file, message: "Field '#{name}': [#{ref}]@d dimension was not found in the project"}
|
|
231
|
+
end
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
parsed.field_refs.each do |ref|
|
|
235
|
+
types = fields[ref]
|
|
236
|
+
if types.nil?
|
|
237
|
+
failures << {file: file, message: "Field '#{name}': [#{ref}] field was not found in the project"}
|
|
238
|
+
elsif types.size > 1
|
|
239
|
+
# references.rb:97
|
|
240
|
+
failures << {
|
|
241
|
+
file: file,
|
|
242
|
+
message: "Field '#{name}': [#{ref}] matched multiple fields. Use @m or @d to specify the field further."
|
|
243
|
+
}
|
|
244
|
+
elsif types.include?("measure")
|
|
245
|
+
measure_refs << ref
|
|
246
|
+
end
|
|
247
|
+
end
|
|
248
|
+
|
|
249
|
+
# references.rb:127-131
|
|
250
|
+
if dimension && measure_refs.any?
|
|
251
|
+
failures << {
|
|
252
|
+
file: file,
|
|
253
|
+
message: "Field '#{name}': dimensions cannot reference measures. These are measures: #{measure_refs.join(", ")}"
|
|
254
|
+
}
|
|
255
|
+
end
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
def expression_sql(expression)
|
|
259
|
+
sql = expression.is_a?(Hash) ? (expression["sql"] || expression[:sql]) : expression
|
|
260
|
+
sql if sql.is_a?(String) && !sql.strip.empty?
|
|
261
|
+
end
|
|
262
|
+
|
|
263
|
+
# ---- table-level: snapshots and partitions ----
|
|
264
|
+
|
|
265
|
+
def validate_table_semantics(content, file, failures)
|
|
266
|
+
validate_snapshot_semantics(content, file, failures)
|
|
267
|
+
validate_partition_semantics(content, file, failures)
|
|
268
|
+
end
|
|
269
|
+
|
|
270
|
+
def validate_snapshot_semantics(content, file, failures)
|
|
271
|
+
snap_name = content["snapshot"].to_s.downcase.strip
|
|
272
|
+
|
|
273
|
+
if snap_name.empty?
|
|
274
|
+
# expression.rb:39-43 — a snapshot measure only maps to snapshot tables.
|
|
275
|
+
Array(content["fields"]).each do |f|
|
|
276
|
+
next unless f.is_a?(Hash)
|
|
277
|
+
next unless SNAPSHOT_TYPES.include?(f["snapshot"].to_s)
|
|
278
|
+
next if f["type"].to_s.downcase.strip == "dimension"
|
|
279
|
+
|
|
280
|
+
failures << {
|
|
281
|
+
file: file,
|
|
282
|
+
message: "Field '#{f["name"]}' is a snapshot measure and can only be mapped to snapshot tables " \
|
|
283
|
+
"(add a table-level 'snapshot' key naming a date dimension)"
|
|
284
|
+
}
|
|
285
|
+
end
|
|
286
|
+
return
|
|
287
|
+
end
|
|
288
|
+
|
|
289
|
+
# deployer/handlers/table.rb:204 + table/snappable.rb:40-43
|
|
290
|
+
unless (project_semantic_index[:fields][snap_name] || []).include?("dimension")
|
|
291
|
+
failures << {file: file, message: "Snapshot dimension '#{content["snapshot"]}' was not found in the project"}
|
|
292
|
+
return
|
|
293
|
+
end
|
|
294
|
+
|
|
295
|
+
data_type = project_semantic_index[:dimension_data_types][snap_name].to_s
|
|
296
|
+
unless %w[date date_time].include?(data_type)
|
|
297
|
+
failures << {
|
|
298
|
+
file: file,
|
|
299
|
+
message: "Snapshot date '#{content["snapshot"]}' must be a date or date_time dimension (got '#{data_type}')"
|
|
300
|
+
}
|
|
301
|
+
end
|
|
302
|
+
end
|
|
303
|
+
|
|
304
|
+
def validate_partition_semantics(content, file, failures)
|
|
305
|
+
partitions = content["partition"] || content["partitions"]
|
|
306
|
+
return if partitions.nil?
|
|
307
|
+
|
|
308
|
+
unless partitions.is_a?(Array)
|
|
309
|
+
failures << {file: file, message: "'partitions' must be an array"}
|
|
310
|
+
return
|
|
311
|
+
end
|
|
312
|
+
|
|
313
|
+
partitions.each_with_index do |part, idx|
|
|
314
|
+
unless part.is_a?(Hash)
|
|
315
|
+
failures << {file: file, message: "Partition at index #{idx} must be a mapping"}
|
|
316
|
+
next
|
|
317
|
+
end
|
|
318
|
+
|
|
319
|
+
label = part["dimension"] || "index #{idx}"
|
|
320
|
+
dim_name = part["dimension"].to_s.downcase.strip
|
|
321
|
+
if dim_name.empty?
|
|
322
|
+
failures << {file: file, message: "Partition at index #{idx} missing 'dimension'"}
|
|
323
|
+
elsif !(project_semantic_index[:fields][dim_name] || []).include?("dimension")
|
|
324
|
+
failures << {file: file, message: "Partition dimension '#{part["dimension"]}' was not found in the project"}
|
|
325
|
+
end
|
|
326
|
+
|
|
327
|
+
predicate = part["predicate"]
|
|
328
|
+
if predicate && !PARTITION_PREDICATES.include?(predicate.to_s)
|
|
329
|
+
failures << {
|
|
330
|
+
file: file,
|
|
331
|
+
message: "Partition on '#{label}' has invalid predicate '#{predicate}' (expected one of: #{PARTITION_PREDICATES.join(", ")})"
|
|
332
|
+
}
|
|
333
|
+
end
|
|
334
|
+
|
|
335
|
+
validate_partition_values(part, dim_name, label, predicate, file, failures)
|
|
336
|
+
end
|
|
337
|
+
end
|
|
338
|
+
|
|
339
|
+
# semantic/partition.rb:15-18 + parseable_date.rb
|
|
340
|
+
def validate_partition_values(part, dim_name, label, predicate, file, failures)
|
|
341
|
+
data_type = project_semantic_index[:dimension_data_types][dim_name].to_s
|
|
342
|
+
values = [part["filter_value"]]
|
|
343
|
+
values << part["filter_value_end"] if predicate.to_s == "between"
|
|
344
|
+
|
|
345
|
+
values.each_with_index do |value, i|
|
|
346
|
+
key = i.zero? ? "filter_value" : "filter_value_end"
|
|
347
|
+
|
|
348
|
+
if value.to_s.strip.empty?
|
|
349
|
+
failures << {file: file, message: "Partition on '#{label}' missing '#{key}'"}
|
|
350
|
+
elsif NUMERIC_DATA_TYPES.include?(data_type) && !numeric?(value)
|
|
351
|
+
failures << {file: file, message: "Partition on '#{label}' #{key} '#{value}' is not a number"}
|
|
352
|
+
elsif %w[date date_time].include?(data_type) && !parseable_partition_date?(value)
|
|
353
|
+
failures << {
|
|
354
|
+
file: file,
|
|
355
|
+
message: "Partition on '#{label}' #{key} '#{value}' is not a dynamic date (like 28d, 6h, 1m) or a parseable date"
|
|
356
|
+
}
|
|
357
|
+
end
|
|
358
|
+
end
|
|
359
|
+
end
|
|
360
|
+
|
|
361
|
+
def numeric?(value)
|
|
362
|
+
Float(value)
|
|
363
|
+
true
|
|
364
|
+
rescue ArgumentError, TypeError
|
|
365
|
+
false
|
|
366
|
+
end
|
|
367
|
+
|
|
368
|
+
def parseable_partition_date?(value)
|
|
369
|
+
return true if value.is_a?(Date) || value.is_a?(Time)
|
|
370
|
+
|
|
371
|
+
s = value.to_s.strip
|
|
372
|
+
return true if DYNAMIC_DATE.match?(s)
|
|
373
|
+
|
|
374
|
+
DateTime.parse(s)
|
|
375
|
+
true
|
|
376
|
+
rescue ArgumentError, TypeError
|
|
377
|
+
false
|
|
378
|
+
end
|
|
379
|
+
|
|
380
|
+
# ---- relationships (semantic/join_def.rb, deployer/handlers/relationships.rb) ----
|
|
381
|
+
|
|
382
|
+
def validate_relationship_semantics(content, file, failures)
|
|
383
|
+
ds_key = resolve_datasource_value(content["datasource"])
|
|
384
|
+
|
|
385
|
+
content.each do |key, definition|
|
|
386
|
+
next if key == "datasource" || key == "imports"
|
|
387
|
+
next unless definition.is_a?(Hash)
|
|
388
|
+
|
|
389
|
+
sql = definition["sql"].to_s.downcase
|
|
390
|
+
# join_def.rb:70-77
|
|
391
|
+
if !sql.empty? && !(sql.match?(/left\./) && sql.match?(/right\./))
|
|
392
|
+
failures << {file: file, message: "Relationship '#{key}' join sql should be of the format left.column = right.column"}
|
|
393
|
+
end
|
|
394
|
+
|
|
395
|
+
if definition.key?("join") && !JOIN_TYPES.include?(definition["join"].to_s)
|
|
396
|
+
failures << {
|
|
397
|
+
file: file,
|
|
398
|
+
message: "Relationship '#{key}' has invalid join '#{definition["join"]}' (expected one of: #{JOIN_TYPES.join(", ")})"
|
|
399
|
+
}
|
|
400
|
+
end
|
|
401
|
+
|
|
402
|
+
validate_relationship_tables(key, definition, ds_key, file, failures)
|
|
403
|
+
end
|
|
404
|
+
end
|
|
405
|
+
|
|
406
|
+
def validate_relationship_tables(key, definition, ds_key, file, failures)
|
|
407
|
+
tables = project_semantic_index[:tables]
|
|
408
|
+
left = definition["left"].to_s.downcase.strip
|
|
409
|
+
right = definition["right"].to_s.downcase.strip
|
|
410
|
+
|
|
411
|
+
[["left", left], ["right", right]].each do |side, table_name|
|
|
412
|
+
next if table_name.empty?
|
|
413
|
+
|
|
414
|
+
table = tables[table_name]
|
|
415
|
+
if table.nil?
|
|
416
|
+
# deployer/handlers/relationships.rb:106-107
|
|
417
|
+
failures << {file: file, message: "Relationship '#{key}': #{side} table '#{definition[side]}' was not found in the project"}
|
|
418
|
+
elsif ds_key && (table_ds = resolve_datasource_value(table["datasource"])) && table_ds != ds_key
|
|
419
|
+
# join_def.rb:65-67
|
|
420
|
+
failures << {
|
|
421
|
+
file: file,
|
|
422
|
+
message: "Relationship '#{key}': #{side} table '#{definition[side]}' is not in datasource '#{ds_key}' (tables can only join within one datasource)"
|
|
423
|
+
}
|
|
424
|
+
end
|
|
425
|
+
end
|
|
426
|
+
|
|
427
|
+
return if left.empty? || right.empty?
|
|
428
|
+
|
|
429
|
+
# join_def.rb:79-83 — joins are bi-directional, so either order collides.
|
|
430
|
+
pair = [left, right].sort
|
|
431
|
+
if (existing = seen_join_pairs[pair])
|
|
432
|
+
failures << {
|
|
433
|
+
file: file,
|
|
434
|
+
message: "Relationship '#{key}': a join between '#{definition["left"]}' and '#{definition["right"]}' already exists (#{existing})"
|
|
435
|
+
}
|
|
436
|
+
else
|
|
437
|
+
seen_join_pairs[pair] = "#{file}: #{key}"
|
|
438
|
+
end
|
|
439
|
+
end
|
|
440
|
+
|
|
441
|
+
def seen_join_pairs
|
|
442
|
+
@seen_join_pairs ||= {}
|
|
443
|
+
end
|
|
444
|
+
|
|
445
|
+
# ---- shared infrastructure ----
|
|
446
|
+
|
|
447
|
+
# Name index of every table and field declared across the project's
|
|
448
|
+
# tbl.*.yml files, keyed by lowercased name (the server resolves
|
|
449
|
+
# references by lower(name), and the expression parser downcases its
|
|
450
|
+
# input, so lowercase keys line up on both sides).
|
|
451
|
+
def project_semantic_index
|
|
452
|
+
@project_semantic_index ||= begin
|
|
453
|
+
fields = {}
|
|
454
|
+
dimension_data_types = {}
|
|
455
|
+
tables = {}
|
|
456
|
+
|
|
457
|
+
Dir.glob("models/**/*.yml").each do |file|
|
|
458
|
+
next unless File.basename(file).start_with?("tbl.")
|
|
459
|
+
|
|
460
|
+
content = begin
|
|
461
|
+
Utils::YamlImportResolver.resolve(file, Dir.pwd)
|
|
462
|
+
rescue
|
|
463
|
+
next # unreadable files are reported by the structural checks
|
|
464
|
+
end
|
|
465
|
+
next unless content.is_a?(Hash)
|
|
466
|
+
|
|
467
|
+
table_name = content["name"].to_s.downcase.strip
|
|
468
|
+
unless table_name.empty?
|
|
469
|
+
tables[table_name] = {"datasource" => content["datasource"], "file" => file}
|
|
470
|
+
end
|
|
471
|
+
|
|
472
|
+
Array(content["fields"]).each do |f|
|
|
473
|
+
next unless f.is_a?(Hash) && f["name"]
|
|
474
|
+
|
|
475
|
+
# Mirror the deployer's dispatch: literal "dimension" or bust.
|
|
476
|
+
type = (f["type"].to_s.downcase.strip == "dimension") ? "dimension" : "measure"
|
|
477
|
+
name = f["name"].to_s.downcase.strip
|
|
478
|
+
entry = (fields[name] ||= [])
|
|
479
|
+
entry << type unless entry.include?(type)
|
|
480
|
+
dimension_data_types[name] = f["data_type"].to_s if type == "dimension"
|
|
481
|
+
end
|
|
482
|
+
end
|
|
483
|
+
|
|
484
|
+
{fields: fields, tables: tables, dimension_data_types: dimension_data_types}
|
|
485
|
+
end
|
|
486
|
+
end
|
|
487
|
+
|
|
488
|
+
# Reserved keywords + aggregate function names for the adapter behind a
|
|
489
|
+
# table's datasource — class-level data from the dwh gem, no connection.
|
|
490
|
+
# Unknown datasources/adapters fall back to the gem's base lists (the
|
|
491
|
+
# broken datasource reference itself is reported elsewhere).
|
|
492
|
+
def adapter_meta(ds_value)
|
|
493
|
+
@adapter_meta ||= {}
|
|
494
|
+
key = resolve_datasource_value(ds_value)
|
|
495
|
+
config = (key && (datasources[key] || datasources[key.to_s.downcase.strip])) || {}
|
|
496
|
+
sym = config["adapter"].to_s.downcase.strip.to_sym
|
|
497
|
+
|
|
498
|
+
@adapter_meta[sym] ||= if DWH.adapter?(sym)
|
|
499
|
+
klass = DWH.get_adapter(sym)
|
|
500
|
+
{reserved: klass.reserved_keywords, aggregates: klass.aggregate_functions}
|
|
501
|
+
else
|
|
502
|
+
{reserved: DWH.reserved_keywords, aggregates: DWH.aggregate_functions}
|
|
503
|
+
end
|
|
504
|
+
end
|
|
505
|
+
end
|
|
506
|
+
end
|
|
507
|
+
end
|
|
@@ -5,6 +5,7 @@ require_relative "../terminal"
|
|
|
5
5
|
require_relative "../credentials"
|
|
6
6
|
require_relative "../output"
|
|
7
7
|
require_relative "../helpers/datasource_helper"
|
|
8
|
+
require_relative "../helpers/semantic_audit_checks"
|
|
8
9
|
require_relative "../agent_mode"
|
|
9
10
|
require_relative "../utils/yaml_import_resolver"
|
|
10
11
|
require_relative "../utils/import_manager"
|
|
@@ -20,12 +21,13 @@ module Strata
|
|
|
20
21
|
include Terminal
|
|
21
22
|
include Output
|
|
22
23
|
include DatasourceHelper
|
|
24
|
+
include SemanticAuditChecks
|
|
23
25
|
include AgentMode
|
|
24
26
|
|
|
25
27
|
REQUIRED_KEYS_FOR_TABLE_MODEL = %w[name physical_name fields datasource].freeze
|
|
26
28
|
REQUIRED_KEYS_FOR_RELATIONSHIP_MODEL = ["datasource"].freeze
|
|
27
29
|
REQUIRED_KEYS_FOR_RELATIONSHIP_DEFINITION = %w[left right sql cardinality].freeze
|
|
28
|
-
RELATIONSHIP_CARDINALITIES = %w[one_to_one one_to_many many_to_one
|
|
30
|
+
RELATIONSHIP_CARDINALITIES = %w[one_to_one one_to_many many_to_one].freeze
|
|
29
31
|
VALID_FORMAT_TYPES = %w[raw number currency percent date datetime html javascript].freeze
|
|
30
32
|
ALLOWED_FIELD_KEYS = %w[
|
|
31
33
|
type name description hidden grains data_type display_type format
|
|
@@ -33,6 +35,7 @@ module Strata
|
|
|
33
35
|
exclusions inclusions extended_blend_group synonyms expression tags
|
|
34
36
|
].freeze
|
|
35
37
|
ALLOWED_EXPRESSION_KEYS = %w[sql array lookup primary_key].freeze
|
|
38
|
+
STRAY_SCRIPT_EXTENSIONS = %w[.rb .py .sh .ipynb].freeze
|
|
36
39
|
EXPRESSION_BOOLEAN_KEYS = %w[array lookup primary_key].freeze
|
|
37
40
|
|
|
38
41
|
# Set default command so `strata audit` still works as `strata audit all`
|
|
@@ -45,6 +48,7 @@ module Strata
|
|
|
45
48
|
results[:models] = run_check("Checking model definitions") { audit_models }
|
|
46
49
|
results[:connections] = run_check("Checking data source connections") { audit_connections }
|
|
47
50
|
|
|
51
|
+
warn_stray_files
|
|
48
52
|
report_results(results)
|
|
49
53
|
end
|
|
50
54
|
|
|
@@ -77,6 +81,17 @@ module Strata
|
|
|
77
81
|
end
|
|
78
82
|
end
|
|
79
83
|
|
|
84
|
+
# A Strata project holds YAML, not scripts. Agents sometimes leave scratch
|
|
85
|
+
# validators behind; warn, don't fail — a helper script is the user's call.
|
|
86
|
+
def warn_stray_files
|
|
87
|
+
strays = Dir.glob("models/**/*").select { File.file?(it) && File.extname(it) != ".yml" }
|
|
88
|
+
strays += Dir.glob("*").select { File.file?(it) && STRAY_SCRIPT_EXTENSIONS.include?(File.extname(it)) }
|
|
89
|
+
return if strays.empty?
|
|
90
|
+
|
|
91
|
+
print_warning("\n Unexpected files — models belong in YAML; delete these or move them out of the project:")
|
|
92
|
+
strays.sort.each { print_warning(" #{it}") }
|
|
93
|
+
end
|
|
94
|
+
|
|
80
95
|
def run_check(message)
|
|
81
96
|
failures = []
|
|
82
97
|
with_spinner(message) do
|
|
@@ -107,7 +122,7 @@ module Strata
|
|
|
107
122
|
end
|
|
108
123
|
|
|
109
124
|
def audit_models
|
|
110
|
-
failures =
|
|
125
|
+
failures = audit_datasource_definitions
|
|
111
126
|
Dir.glob("models/**/*.yml").each do |file|
|
|
112
127
|
audit_imports(file, failures)
|
|
113
128
|
audit_model_file(file, failures)
|
|
@@ -154,13 +169,15 @@ module Strata
|
|
|
154
169
|
def validate_table_model(content, file, failures)
|
|
155
170
|
validate_required_keys(content, file, REQUIRED_KEYS_FOR_TABLE_MODEL, failures)
|
|
156
171
|
validate_datasource_reference(content, file, failures)
|
|
157
|
-
validate_table_fields(content["fields"], file, failures)
|
|
172
|
+
validate_table_fields(content["fields"], file, failures, adapter_meta(content["datasource"]))
|
|
173
|
+
validate_table_semantics(content, file, failures)
|
|
158
174
|
end
|
|
159
175
|
|
|
160
176
|
def validate_relationship_model(content, file, failures)
|
|
161
177
|
validate_required_keys(content, file, REQUIRED_KEYS_FOR_RELATIONSHIP_MODEL, failures)
|
|
162
178
|
validate_datasource_reference(content, file, failures)
|
|
163
179
|
validate_relationship_definitions(content, file, failures)
|
|
180
|
+
validate_relationship_semantics(content, file, failures)
|
|
164
181
|
end
|
|
165
182
|
|
|
166
183
|
def validate_datasource_reference(content, file, failures)
|
|
@@ -180,7 +197,7 @@ module Strata
|
|
|
180
197
|
failures << {file: file, message: "Missing required keys: #{missing.join(", ")}"} if missing.any?
|
|
181
198
|
end
|
|
182
199
|
|
|
183
|
-
def validate_table_fields(fields, file, failures)
|
|
200
|
+
def validate_table_fields(fields, file, failures, adapter_meta)
|
|
184
201
|
unless fields.is_a?(Array)
|
|
185
202
|
failures << {file: file, message: "'fields' must be an array"}
|
|
186
203
|
return
|
|
@@ -197,11 +214,11 @@ module Strata
|
|
|
197
214
|
next
|
|
198
215
|
end
|
|
199
216
|
|
|
200
|
-
validate_field_definition(field, file, idx, failures)
|
|
217
|
+
validate_field_definition(field, file, idx, failures, adapter_meta)
|
|
201
218
|
end
|
|
202
219
|
end
|
|
203
220
|
|
|
204
|
-
def validate_field_definition(field, file, idx, failures)
|
|
221
|
+
def validate_field_definition(field, file, idx, failures, adapter_meta)
|
|
205
222
|
field_name = field["name"] || "index #{idx}"
|
|
206
223
|
|
|
207
224
|
unknown_keys = field.keys.map(&:to_s) - ALLOWED_FIELD_KEYS
|
|
@@ -213,6 +230,7 @@ module Strata
|
|
|
213
230
|
end
|
|
214
231
|
|
|
215
232
|
validate_field_expression(field_name, field["expression"], file, failures)
|
|
233
|
+
validate_field_semantics(field, file, failures, adapter_meta)
|
|
216
234
|
|
|
217
235
|
return unless field.key?("format")
|
|
218
236
|
|
|
@@ -7,6 +7,7 @@ require_relative "../output"
|
|
|
7
7
|
require_relative "../utils/git"
|
|
8
8
|
require "tty-prompt"
|
|
9
9
|
require_relative "../helpers/datasource_helper"
|
|
10
|
+
require_relative "../helpers/prompts"
|
|
10
11
|
require_relative "../helpers/description_helper"
|
|
11
12
|
require_relative "../agent_mode"
|
|
12
13
|
|
|
@@ -62,8 +63,6 @@ module Strata
|
|
|
62
63
|
desc "add [ADAPTER]", "Add a new datasource interactively"
|
|
63
64
|
long_desc_from_file "datasource/add"
|
|
64
65
|
def add(adapter_name = nil)
|
|
65
|
-
prompt = TTY::Prompt.new
|
|
66
|
-
|
|
67
66
|
if adapter_name && !DWH.adapters.keys.map(&:to_s).include?(adapter_name)
|
|
68
67
|
say "Error: '#{adapter_name}' is not a supported adapter", :red
|
|
69
68
|
say "Supported adapters: #{DWH.adapters.keys.join(", ")}", :yellow
|
|
@@ -132,6 +131,11 @@ module Strata
|
|
|
132
131
|
end
|
|
133
132
|
|
|
134
133
|
say "\n✔ Datasource '#{ds_key}' is ready!", :green
|
|
134
|
+
|
|
135
|
+
# 'strata init' runs this command inline and prints its own next steps after.
|
|
136
|
+
return if options["from_init"] || agent_mode?
|
|
137
|
+
|
|
138
|
+
say "\n#{Prompts.build_model_next_steps}", :cyan
|
|
135
139
|
end
|
|
136
140
|
|
|
137
141
|
desc "auth DS_KEY", "Set credentials for the given datasource key (DS_KEY)."
|
|
@@ -182,8 +186,7 @@ module Strata
|
|
|
182
186
|
method_option :catalog, aliases: "c", type: :string, desc: "Change the catalog from the configured one."
|
|
183
187
|
method_option :schema, aliases: "s", type: :string, desc: "Change the schema from the configured one."
|
|
184
188
|
def tables(ds_key = nil)
|
|
185
|
-
|
|
186
|
-
ds_key = resolve_datasource(ds_key, prompt: prompt)
|
|
189
|
+
ds_key = resolve_datasource(ds_key)
|
|
187
190
|
return unless ds_key
|
|
188
191
|
|
|
189
192
|
adapter = create_adapter(ds_key)
|