strata-cli 0.1.20 → 0.1.22
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +56 -0
- data/README.md +26 -9
- data/lib/strata/cli/configuration.rb +11 -1
- data/lib/strata/cli/descriptions/init.txt +8 -7
- data/lib/strata/cli/generators/project.rb +6 -129
- data/lib/strata/cli/generators/templates/AGENTS.md +34 -34
- data/lib/strata/cli/generators/templates/project.yml +2 -2
- data/lib/strata/cli/generators/templates/table.table_name.yml +1 -1
- data/lib/strata/cli/helpers/datasource_helper.rb +19 -2
- data/lib/strata/cli/helpers/project_helper.rb +3 -2
- data/lib/strata/cli/helpers/prompts.rb +2 -2
- data/lib/strata/cli/helpers/semantic_audit_checks.rb +87 -29
- data/lib/strata/cli/main.rb +5 -7
- data/lib/strata/cli/sub_commands/audit.rb +94 -27
- data/lib/strata/cli/sub_commands/datasource.rb +6 -0
- data/lib/strata/cli/sub_commands/deploy.rb +1 -30
- data/lib/strata/cli/sub_commands/project.rb +6 -9
- data/lib/strata/cli/utils/archive.rb +3 -1
- data/lib/strata/cli/utils/sql_expression_parser.rb +53 -12
- data/lib/strata/cli/version.rb +1 -1
- metadata +2 -2
|
@@ -13,6 +13,7 @@ module Strata
|
|
|
13
13
|
#
|
|
14
14
|
# Server-only checks (universe/path integrity, physical table existence
|
|
15
15
|
# in the warehouse, licenses) are deliberately not replicated here.
|
|
16
|
+
|
|
16
17
|
module SemanticAuditChecks
|
|
17
18
|
# deployer/handlers/table.rb dispatches on the literal string
|
|
18
19
|
# "dimension"; any other value silently becomes a measure and a
|
|
@@ -24,7 +25,7 @@ module Strata
|
|
|
24
25
|
DISPLAY_TYPES = %w[default html url email phone_number image].freeze
|
|
25
26
|
DATE_GRAINS = %w[raw millisecond second minute hour day month week quarter year].freeze
|
|
26
27
|
NUMERIC_DATA_TYPES = %w[integer decimal bigint].freeze
|
|
27
|
-
# semantic/measure.rb + measure/excludable.rb + measure/includable.rb
|
|
28
|
+
# semantic/measure.rb + semantic/measure/excludable.rb + semantic/measure/includable.rb
|
|
28
29
|
SNAPSHOT_TYPES = %w[beginning ending].freeze
|
|
29
30
|
EXCLUSION_TYPES = %w[exclude exclude_all_except exclude_all].freeze
|
|
30
31
|
EXCLUSION_ENTITY_TYPES = %w[dimension table universe].freeze
|
|
@@ -34,8 +35,8 @@ module Strata
|
|
|
34
35
|
DYNAMIC_DATE = /(\d{1,3})([hdwmqy])/i
|
|
35
36
|
# semantic/join_def.rb enums — note: no many_to_many.
|
|
36
37
|
JOIN_TYPES = %w[inner left right].freeze
|
|
37
|
-
# semantic/expression.rb:17 —
|
|
38
|
-
MEASURE_SQL_FORMAT = /\A.*\w+\(.*\).*\z/
|
|
38
|
+
# semantic/expression.rb:17 — any word(...) call. /m so a block scalar or a measure spanning lines still matches.
|
|
39
|
+
MEASURE_SQL_FORMAT = /\A.*\w+\(.*\).*\z/im
|
|
39
40
|
|
|
40
41
|
private
|
|
41
42
|
|
|
@@ -65,9 +66,11 @@ module Strata
|
|
|
65
66
|
next if name.empty?
|
|
66
67
|
|
|
67
68
|
if seen_names[name]
|
|
69
|
+
# handlers/datasource.rb:18-20 merges them onto one row, so tables
|
|
70
|
+
# silently repoint at the other warehouse.
|
|
68
71
|
failures << {
|
|
69
72
|
file: "datasources.yml",
|
|
70
|
-
message: "Datasource '#{key}' duplicates the name '#{config["name"]}' used by '#{seen_names[name]}'"
|
|
73
|
+
message: "Datasource '#{key}' duplicates the name '#{config["name"]}' used by '#{seen_names[name]}' (on deploy they merge into one datasource)"
|
|
71
74
|
}
|
|
72
75
|
else
|
|
73
76
|
seen_names[name] = key
|
|
@@ -82,6 +85,7 @@ module Strata
|
|
|
82
85
|
def validate_field_semantics(field, file, failures, meta)
|
|
83
86
|
name = field["name"]
|
|
84
87
|
|
|
88
|
+
# handlers/table.rb:113 turns anything but "dimension" into a Measure.
|
|
85
89
|
unless FIELD_TYPES.include?(field["type"].to_s.downcase.strip)
|
|
86
90
|
failures << {
|
|
87
91
|
file: file,
|
|
@@ -90,10 +94,29 @@ module Strata
|
|
|
90
94
|
end
|
|
91
95
|
|
|
92
96
|
validate_field_enums(name, field, file, failures)
|
|
93
|
-
|
|
97
|
+
if field["type"].to_s.downcase.strip == "dimension"
|
|
98
|
+
validate_no_measure_keys(name, field, file, failures)
|
|
99
|
+
else
|
|
100
|
+
validate_measure_rules(name, field, file, failures)
|
|
101
|
+
end
|
|
94
102
|
validate_expression_semantics(name, field, file, failures, meta)
|
|
95
103
|
end
|
|
96
104
|
|
|
105
|
+
# Written to a dimension by handlers/table.rb:119 but untyped, so
|
|
106
|
+
# `snapshot: ending` stores 0 and reads back as `beginning`.
|
|
107
|
+
MEASURE_ONLY_KEYS = %w[snapshot exclusion_type exclusions inclusions].freeze
|
|
108
|
+
|
|
109
|
+
def validate_no_measure_keys(name, field, file, failures)
|
|
110
|
+
present = MEASURE_ONLY_KEYS.select { field.key?(it) }
|
|
111
|
+
return if present.empty?
|
|
112
|
+
|
|
113
|
+
failures << {
|
|
114
|
+
file: file,
|
|
115
|
+
message: "Field '#{name}' is a dimension, so #{present.sort.join(", ")} #{(present.size == 1) ? "does" : "do"} not apply " \
|
|
116
|
+
"(#{(present.size == 1) ? "it is" : "they are"} measure-only; the server stores the value untyped and never reads it)"
|
|
117
|
+
}
|
|
118
|
+
end
|
|
119
|
+
|
|
97
120
|
def validate_field_enums(name, field, file, failures)
|
|
98
121
|
data_type = field["data_type"].to_s.strip
|
|
99
122
|
if data_type.empty?
|
|
@@ -105,7 +128,8 @@ module Strata
|
|
|
105
128
|
}
|
|
106
129
|
end
|
|
107
130
|
|
|
108
|
-
|
|
131
|
+
# field.rb:16 is a nullable enum: an empty key is an absent one.
|
|
132
|
+
if !field["display_type"].to_s.strip.empty? && !DISPLAY_TYPES.include?(field["display_type"].to_s)
|
|
109
133
|
failures << {
|
|
110
134
|
file: file,
|
|
111
135
|
message: "Field '#{name}' has invalid display_type '#{field["display_type"]}' (expected one of: #{DISPLAY_TYPES.join(", ")})"
|
|
@@ -119,7 +143,8 @@ module Strata
|
|
|
119
143
|
end
|
|
120
144
|
|
|
121
145
|
def validate_measure_rules(name, field, file, failures)
|
|
122
|
-
|
|
146
|
+
# measure.rb:6 is `enum :snapshot, [...], validate: {allow_nil: true}`.
|
|
147
|
+
if !field["snapshot"].to_s.strip.empty? && !SNAPSHOT_TYPES.include?(field["snapshot"].to_s)
|
|
123
148
|
failures << {
|
|
124
149
|
file: file,
|
|
125
150
|
message: "Field '#{name}' has invalid snapshot '#{field["snapshot"]}' (expected beginning or ending)"
|
|
@@ -139,6 +164,16 @@ module Strata
|
|
|
139
164
|
failures << {file: file, message: "Field '#{name}' exclusions should be empty when exclusion_type is exclude_all"}
|
|
140
165
|
end
|
|
141
166
|
|
|
167
|
+
# excludable.rb:9,12 gates the rules on exclusion_type, so without it they
|
|
168
|
+
# are stored and never applied.
|
|
169
|
+
if exclusions.any? && exclusion_type.to_s.strip.empty?
|
|
170
|
+
failures << {
|
|
171
|
+
file: file,
|
|
172
|
+
message: "Field '#{name}' has exclusions but no 'exclusion_type', so the server stores the rules and never applies them"
|
|
173
|
+
}
|
|
174
|
+
exclusions = []
|
|
175
|
+
end
|
|
176
|
+
|
|
142
177
|
exclusions.each_with_index do |rule, idx|
|
|
143
178
|
unless rule.is_a?(Hash)
|
|
144
179
|
failures << {file: file, message: "Field '#{name}' exclusion at index #{idx} must be a mapping"}
|
|
@@ -181,14 +216,14 @@ module Strata
|
|
|
181
216
|
end
|
|
182
217
|
end
|
|
183
218
|
|
|
184
|
-
# ---- expressions (semantic/expression.rb, expression/references.rb) ----
|
|
219
|
+
# ---- expressions (semantic/expression.rb, semantic/expression/references.rb) ----
|
|
185
220
|
|
|
186
221
|
def validate_expression_semantics(name, field, file, failures, meta)
|
|
187
222
|
sql = expression_sql(field["expression"])
|
|
188
223
|
return unless sql
|
|
189
224
|
|
|
190
225
|
begin
|
|
191
|
-
parsed = Sql::ExpressionParser.new(sql, reserved: meta[:reserved], aggregates: meta[:aggregates])
|
|
226
|
+
parsed = Sql::ExpressionParser.new(sql, reserved: meta[:reserved], aggregates: meta[:aggregates], quote: meta[:quote])
|
|
192
227
|
rescue => e
|
|
193
228
|
failures << {file: file, message: "Field '#{name}': expression could not be parsed: #{e.message}"}
|
|
194
229
|
return
|
|
@@ -236,7 +271,7 @@ module Strata
|
|
|
236
271
|
if types.nil?
|
|
237
272
|
failures << {file: file, message: "Field '#{name}': [#{ref}] field was not found in the project"}
|
|
238
273
|
elsif types.size > 1
|
|
239
|
-
# references.rb:
|
|
274
|
+
# references.rb:98
|
|
240
275
|
failures << {
|
|
241
276
|
file: file,
|
|
242
277
|
message: "Field '#{name}': [#{ref}] matched multiple fields. Use @m or @d to specify the field further."
|
|
@@ -246,7 +281,7 @@ module Strata
|
|
|
246
281
|
end
|
|
247
282
|
end
|
|
248
283
|
|
|
249
|
-
# references.rb:
|
|
284
|
+
# references.rb:126-132
|
|
250
285
|
if dimension && measure_refs.any?
|
|
251
286
|
failures << {
|
|
252
287
|
file: file,
|
|
@@ -286,7 +321,7 @@ module Strata
|
|
|
286
321
|
return
|
|
287
322
|
end
|
|
288
323
|
|
|
289
|
-
# deployer/handlers/table.rb:204 + table/snappable.rb:
|
|
324
|
+
# deployer/handlers/table.rb:204 + table/snappable.rb:38-43
|
|
290
325
|
unless (project_semantic_index[:fields][snap_name] || []).include?("dimension")
|
|
291
326
|
failures << {file: file, message: "Snapshot dimension '#{content["snapshot"]}' was not found in the project"}
|
|
292
327
|
return
|
|
@@ -294,6 +329,8 @@ module Strata
|
|
|
294
329
|
|
|
295
330
|
data_type = project_semantic_index[:dimension_data_types][snap_name].to_s
|
|
296
331
|
unless %w[date date_time].include?(data_type)
|
|
332
|
+
# snappable.rb:38-43, which skip_reference_validation keeps off the
|
|
333
|
+
# deploy path -- snapshot_visitor then returns a wrong number silently.
|
|
297
334
|
failures << {
|
|
298
335
|
file: file,
|
|
299
336
|
message: "Snapshot date '#{content["snapshot"]}' must be a date or date_time dimension (got '#{data_type}')"
|
|
@@ -388,11 +425,13 @@ module Strata
|
|
|
388
425
|
|
|
389
426
|
sql = definition["sql"].to_s.downcase
|
|
390
427
|
# join_def.rb:70-77
|
|
391
|
-
if !sql.empty? && !(sql.
|
|
428
|
+
if !sql.empty? && !(sql.include?("left.") && sql.include?("right."))
|
|
392
429
|
failures << {file: file, message: "Relationship '#{key}' join sql should be of the format left.column = right.column"}
|
|
393
430
|
end
|
|
394
431
|
|
|
395
|
-
|
|
432
|
+
# relationships.rb:121 downcases and defaults to "inner" first.
|
|
433
|
+
join = definition["join"].to_s.downcase.strip
|
|
434
|
+
if !join.empty? && !JOIN_TYPES.include?(join)
|
|
396
435
|
failures << {
|
|
397
436
|
file: file,
|
|
398
437
|
message: "Relationship '#{key}' has invalid join '#{definition["join"]}' (expected one of: #{JOIN_TYPES.join(", ")})"
|
|
@@ -404,19 +443,19 @@ module Strata
|
|
|
404
443
|
end
|
|
405
444
|
|
|
406
445
|
def validate_relationship_tables(key, definition, ds_key, file, failures)
|
|
407
|
-
|
|
446
|
+
table_datasources = project_semantic_index[:table_datasources]
|
|
408
447
|
left = definition["left"].to_s.downcase.strip
|
|
409
448
|
right = definition["right"].to_s.downcase.strip
|
|
410
449
|
|
|
411
450
|
[["left", left], ["right", right]].each do |side, table_name|
|
|
412
451
|
next if table_name.empty?
|
|
413
452
|
|
|
414
|
-
|
|
415
|
-
if
|
|
453
|
+
declared_in = table_datasources[table_name]
|
|
454
|
+
if declared_in.nil?
|
|
416
455
|
# deployer/handlers/relationships.rb:106-107
|
|
417
456
|
failures << {file: file, message: "Relationship '#{key}': #{side} table '#{definition[side]}' was not found in the project"}
|
|
418
|
-
elsif ds_key && (
|
|
419
|
-
# join_def.rb:65-67
|
|
457
|
+
elsif ds_key && !declared_in.include?(ds_key)
|
|
458
|
+
# join_def.rb:65-67 — the name is taken in some other datasource only.
|
|
420
459
|
failures << {
|
|
421
460
|
file: file,
|
|
422
461
|
message: "Relationship '#{key}': #{side} table '#{definition[side]}' is not in datasource '#{ds_key}' (tables can only join within one datasource)"
|
|
@@ -427,14 +466,18 @@ module Strata
|
|
|
427
466
|
return if left.empty? || right.empty?
|
|
428
467
|
|
|
429
468
|
# join_def.rb:79-83 — joins are bi-directional, so either order collides.
|
|
430
|
-
pair
|
|
469
|
+
# Scoped by datasource: the pair identifies table records.
|
|
470
|
+
pair = [ds_key, *[left, right].sort]
|
|
431
471
|
if (existing = seen_join_pairs[pair])
|
|
472
|
+
# relationships.rb:113-115 raises within a file; across files
|
|
473
|
+
# first_or_initialize lets glob order decide. Non-deterministic either way.
|
|
432
474
|
failures << {
|
|
433
475
|
file: file,
|
|
434
|
-
message: "Relationship '#{key}': a join between '#{definition["left"]}' and '#{definition["right"]}'
|
|
476
|
+
message: "Relationship '#{key}': a join between '#{definition["left"]}' and '#{definition["right"]}' " \
|
|
477
|
+
"already exists (#{existing[:file]}: #{existing[:key]})"
|
|
435
478
|
}
|
|
436
479
|
else
|
|
437
|
-
seen_join_pairs[pair] =
|
|
480
|
+
seen_join_pairs[pair] = {file: file, key: key}
|
|
438
481
|
end
|
|
439
482
|
end
|
|
440
483
|
|
|
@@ -442,19 +485,26 @@ module Strata
|
|
|
442
485
|
@seen_join_pairs ||= {}
|
|
443
486
|
end
|
|
444
487
|
|
|
488
|
+
# Reported without failing the audit.
|
|
489
|
+
def semantic_warnings
|
|
490
|
+
@semantic_warnings ||= []
|
|
491
|
+
end
|
|
492
|
+
|
|
445
493
|
# ---- shared infrastructure ----
|
|
446
494
|
|
|
447
495
|
# Name index of every table and field declared across the project's
|
|
448
496
|
# tbl.*.yml files, keyed by lowercased name (the server resolves
|
|
449
497
|
# references by lower(name), and the expression parser downcases its
|
|
450
498
|
# input, so lowercase keys line up on both sides).
|
|
499
|
+
# Fields stay keyed by name alone (uniqueness scope: [:type, :branch_id]);
|
|
500
|
+
# tables are per-datasource, so that index maps a name to its datasources.
|
|
451
501
|
def project_semantic_index
|
|
452
502
|
@project_semantic_index ||= begin
|
|
453
503
|
fields = {}
|
|
454
504
|
dimension_data_types = {}
|
|
455
|
-
|
|
505
|
+
table_datasources = {}
|
|
456
506
|
|
|
457
|
-
Dir.glob("models/**/*.yml").each do |file|
|
|
507
|
+
Dir.glob("models/**/*.{yml,yaml}").each do |file|
|
|
458
508
|
next unless File.basename(file).start_with?("tbl.")
|
|
459
509
|
|
|
460
510
|
content = begin
|
|
@@ -466,7 +516,9 @@ module Strata
|
|
|
466
516
|
|
|
467
517
|
table_name = content["name"].to_s.downcase.strip
|
|
468
518
|
unless table_name.empty?
|
|
469
|
-
|
|
519
|
+
# Unique per datasource (semantic/table.rb:32), so one name can
|
|
520
|
+
# belong to several.
|
|
521
|
+
(table_datasources[table_name] ||= []) << resolve_datasource_value(content["datasource"])
|
|
470
522
|
end
|
|
471
523
|
|
|
472
524
|
Array(content["fields"]).each do |f|
|
|
@@ -481,7 +533,7 @@ module Strata
|
|
|
481
533
|
end
|
|
482
534
|
end
|
|
483
535
|
|
|
484
|
-
{fields: fields,
|
|
536
|
+
{fields: fields, table_datasources: table_datasources, dimension_data_types: dimension_data_types}
|
|
485
537
|
end
|
|
486
538
|
end
|
|
487
539
|
|
|
@@ -492,14 +544,20 @@ module Strata
|
|
|
492
544
|
def adapter_meta(ds_value)
|
|
493
545
|
@adapter_meta ||= {}
|
|
494
546
|
key = resolve_datasource_value(ds_value)
|
|
495
|
-
config = (key &&
|
|
547
|
+
config = (key && datasources[key]) || {}
|
|
496
548
|
sym = config["adapter"].to_s.downcase.strip.to_sym
|
|
497
549
|
|
|
498
550
|
@adapter_meta[sym] ||= if DWH.adapter?(sym)
|
|
499
551
|
klass = DWH.get_adapter(sym)
|
|
500
|
-
{
|
|
552
|
+
{
|
|
553
|
+
reserved: klass.reserved_keywords,
|
|
554
|
+
aggregates: klass.aggregate_functions,
|
|
555
|
+
# references.rb:5-8 passes the adapter's own identifier quote, so a
|
|
556
|
+
# backtick warehouse reads `order` as a column.
|
|
557
|
+
quote: klass.adapter_settings[:quote]
|
|
558
|
+
}
|
|
501
559
|
else
|
|
502
|
-
{reserved: DWH.reserved_keywords, aggregates: DWH.aggregate_functions}
|
|
560
|
+
{reserved: DWH.reserved_keywords, aggregates: DWH.aggregate_functions, quote: Sql::ExpressionParser::ANSI_QUOTE}
|
|
503
561
|
end
|
|
504
562
|
end
|
|
505
563
|
end
|
data/lib/strata/cli/main.rb
CHANGED
|
@@ -30,21 +30,19 @@ module Strata
|
|
|
30
30
|
say VERSION
|
|
31
31
|
end
|
|
32
32
|
|
|
33
|
-
desc "init
|
|
33
|
+
desc "init PROJECT_NAME", "Initializes a new Strata project."
|
|
34
34
|
long_desc_from_file "init"
|
|
35
35
|
option :datasource, aliases: ["d"], type: :string, desc: "One of the supported data warehouse adapters.",
|
|
36
36
|
repeatable: true
|
|
37
|
-
option :
|
|
38
|
-
option :api_key, aliases: ["a"], type: :string, desc: "Api Key.
|
|
37
|
+
option :server, aliases: ["s"], type: :string, desc: "Strata server URL. Defaults to #{Configuration::DEFAULT_SERVER}."
|
|
38
|
+
option :api_key, aliases: ["a"], type: :string, desc: "Api Key. Optional — the first deploy signs you in via browser."
|
|
39
39
|
option :verbose, aliases: ["v"], type: :boolean, default: false,
|
|
40
40
|
desc: "Show detailed init output (file-by-file actions)."
|
|
41
41
|
|
|
42
42
|
def init(project_name = nil)
|
|
43
|
-
unless project_name
|
|
44
|
-
raise Strata::CommandError, "PROJECT_NAME is required when not using --source option."
|
|
45
|
-
end
|
|
43
|
+
raise Strata::CommandError, "PROJECT_NAME is required." unless project_name
|
|
46
44
|
|
|
47
|
-
print_status(:created,
|
|
45
|
+
print_status(:created, project_name.to_s, type: :info)
|
|
48
46
|
invoke Generators::Project, [project_name], options
|
|
49
47
|
end
|
|
50
48
|
|
|
@@ -24,7 +24,7 @@ module Strata
|
|
|
24
24
|
include SemanticAuditChecks
|
|
25
25
|
include AgentMode
|
|
26
26
|
|
|
27
|
-
REQUIRED_KEYS_FOR_TABLE_MODEL = %w[name physical_name
|
|
27
|
+
REQUIRED_KEYS_FOR_TABLE_MODEL = %w[name physical_name datasource].freeze
|
|
28
28
|
REQUIRED_KEYS_FOR_RELATIONSHIP_MODEL = ["datasource"].freeze
|
|
29
29
|
REQUIRED_KEYS_FOR_RELATIONSHIP_DEFINITION = %w[left right sql cardinality].freeze
|
|
30
30
|
RELATIONSHIP_CARDINALITIES = %w[one_to_one one_to_many many_to_one].freeze
|
|
@@ -35,7 +35,15 @@ module Strata
|
|
|
35
35
|
exclusions inclusions extended_blend_group synonyms expression tags
|
|
36
36
|
].freeze
|
|
37
37
|
ALLOWED_EXPRESSION_KEYS = %w[sql array lookup primary_key].freeze
|
|
38
|
+
# What the deployer reads from a tbl file (handlers/table.rb:4, :62, :71,
|
|
39
|
+
# :152, :202), plus the CLI's own imports.
|
|
40
|
+
ALLOWED_TABLE_KEYS = %w[
|
|
41
|
+
name description physical_name cost metadata tags
|
|
42
|
+
datasource fields snapshot partition partitions imports
|
|
43
|
+
].freeze
|
|
38
44
|
STRAY_SCRIPT_EXTENSIONS = %w[.rb .py .sh .ipynb].freeze
|
|
45
|
+
# deploy_job.rb:121,162 globs models as {yml,yaml}, so both are models.
|
|
46
|
+
MODEL_EXTENSIONS = %w[.yml .yaml].freeze
|
|
39
47
|
EXPRESSION_BOOLEAN_KEYS = %w[array lookup primary_key].freeze
|
|
40
48
|
|
|
41
49
|
# Set default command so `strata audit` still works as `strata audit all`
|
|
@@ -46,8 +54,10 @@ module Strata
|
|
|
46
54
|
results = {}
|
|
47
55
|
results[:yaml] = run_check("Checking YAML syntax") { audit_yaml_syntax }
|
|
48
56
|
results[:models] = run_check("Checking model definitions") { audit_models }
|
|
49
|
-
results[:connections] = run_check("Checking data source connections") { audit_connections }
|
|
57
|
+
results[:connections] = run_check("Checking data source connections") { audit_connections(semantic_warnings) }
|
|
50
58
|
|
|
59
|
+
warn_semantic_warnings
|
|
60
|
+
warn_unverifiable_datasources
|
|
51
61
|
warn_stray_files
|
|
52
62
|
report_results(results)
|
|
53
63
|
end
|
|
@@ -61,12 +71,15 @@ module Strata
|
|
|
61
71
|
desc "models", "Check model definitions and schema structure"
|
|
62
72
|
def models
|
|
63
73
|
results = {models: run_check("Checking model definitions") { audit_models }}
|
|
74
|
+
warn_semantic_warnings
|
|
64
75
|
report_results(results)
|
|
65
76
|
end
|
|
66
77
|
|
|
67
78
|
desc "connections", "Check data source connections"
|
|
68
79
|
def connections
|
|
69
80
|
results = {connections: run_check("Checking data source connections") { audit_connections }}
|
|
81
|
+
warn_semantic_warnings
|
|
82
|
+
warn_unverifiable_datasources
|
|
70
83
|
report_results(results)
|
|
71
84
|
end
|
|
72
85
|
|
|
@@ -84,7 +97,7 @@ module Strata
|
|
|
84
97
|
# A Strata project holds YAML, not scripts. Agents sometimes leave scratch
|
|
85
98
|
# validators behind; warn, don't fail — a helper script is the user's call.
|
|
86
99
|
def warn_stray_files
|
|
87
|
-
strays = Dir.glob("models/**/*").select { File.file?(it) && File.extname(it)
|
|
100
|
+
strays = Dir.glob("models/**/*").select { File.file?(it) && !MODEL_EXTENSIONS.include?(File.extname(it)) }
|
|
88
101
|
strays += Dir.glob("*").select { File.file?(it) && STRAY_SCRIPT_EXTENSIONS.include?(File.extname(it)) }
|
|
89
102
|
return if strays.empty?
|
|
90
103
|
|
|
@@ -92,6 +105,25 @@ module Strata
|
|
|
92
105
|
strays.sort.each { print_warning(" #{it}") }
|
|
93
106
|
end
|
|
94
107
|
|
|
108
|
+
# Problems the server tolerates: shown, but never failed.
|
|
109
|
+
def warn_semantic_warnings
|
|
110
|
+
return if semantic_warnings.empty?
|
|
111
|
+
|
|
112
|
+
print_warning("\n Accepted by the server, but worth a look:")
|
|
113
|
+
semantic_warnings.each { print_warning(" #{it[:file]}: #{it[:message]}") }
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# Pointing a datasource at a path the server owns is a normal setup — the sample
|
|
117
|
+
# project's /data/tpcds.duckdb ships inside the Strata image — so say which
|
|
118
|
+
# connections went unchecked instead of failing a correct project.
|
|
119
|
+
def warn_unverifiable_datasources
|
|
120
|
+
skipped = datasources.select { |_key, config| local_file_missing?(config) }
|
|
121
|
+
return if skipped.empty?
|
|
122
|
+
|
|
123
|
+
print_warning("\n Connection not checked — file not on this machine (the server opens it at deploy):")
|
|
124
|
+
skipped.each { |key, config| print_warning(" #{key}: #{config["file"]}") }
|
|
125
|
+
end
|
|
126
|
+
|
|
95
127
|
def run_check(message)
|
|
96
128
|
failures = []
|
|
97
129
|
with_spinner(message) do
|
|
@@ -108,7 +140,7 @@ module Strata
|
|
|
108
140
|
end
|
|
109
141
|
|
|
110
142
|
def audit_yaml_syntax
|
|
111
|
-
files = Dir.glob("models/**/*.yml") + ["datasources.yml"].select { |f| File.exist?(f) }
|
|
143
|
+
files = Dir.glob("models/**/*.{yml,yaml}") + ["datasources.yml"].select { |f| File.exist?(f) }
|
|
112
144
|
|
|
113
145
|
failures = []
|
|
114
146
|
files.each do |file|
|
|
@@ -123,7 +155,7 @@ module Strata
|
|
|
123
155
|
|
|
124
156
|
def audit_models
|
|
125
157
|
failures = audit_datasource_definitions
|
|
126
|
-
Dir.glob("models/**/*.yml").each do |file|
|
|
158
|
+
Dir.glob("models/**/*.{yml,yaml}").each do |file|
|
|
127
159
|
audit_imports(file, failures)
|
|
128
160
|
audit_model_file(file, failures)
|
|
129
161
|
end
|
|
@@ -160,19 +192,34 @@ module Strata
|
|
|
160
192
|
end
|
|
161
193
|
end
|
|
162
194
|
|
|
195
|
+
# handlers/table.rb:54 drops it, so an unquoted `yes:` does nothing.
|
|
163
196
|
def validate_structure(content, file, failures)
|
|
164
197
|
content.each_key do |k|
|
|
165
|
-
|
|
198
|
+
next if k.is_a?(String)
|
|
199
|
+
|
|
200
|
+
failures << {file: file, message: "Top level key #{k} is not a string"}
|
|
166
201
|
end
|
|
167
202
|
end
|
|
168
203
|
|
|
169
204
|
def validate_table_model(content, file, failures)
|
|
170
205
|
validate_required_keys(content, file, REQUIRED_KEYS_FOR_TABLE_MODEL, failures)
|
|
206
|
+
validate_table_keys(content, file, failures)
|
|
171
207
|
validate_datasource_reference(content, file, failures)
|
|
172
208
|
validate_table_fields(content["fields"], file, failures, adapter_meta(content["datasource"]))
|
|
173
209
|
validate_table_semantics(content, file, failures)
|
|
174
210
|
end
|
|
175
211
|
|
|
212
|
+
# Same slice: `snapshot_date:` deploys a table with no snapshot date.
|
|
213
|
+
def validate_table_keys(content, file, failures)
|
|
214
|
+
unknown_keys = content.keys.select { it.is_a?(String) }.map(&:to_s) - ALLOWED_TABLE_KEYS
|
|
215
|
+
return if unknown_keys.empty?
|
|
216
|
+
|
|
217
|
+
failures << {
|
|
218
|
+
file: file,
|
|
219
|
+
message: "Table has unknown top-level #{(unknown_keys.size == 1) ? "key" : "keys"}: #{unknown_keys.sort.join(", ")}"
|
|
220
|
+
}
|
|
221
|
+
end
|
|
222
|
+
|
|
176
223
|
def validate_relationship_model(content, file, failures)
|
|
177
224
|
validate_required_keys(content, file, REQUIRED_KEYS_FOR_RELATIONSHIP_MODEL, failures)
|
|
178
225
|
validate_datasource_reference(content, file, failures)
|
|
@@ -198,13 +245,14 @@ module Strata
|
|
|
198
245
|
end
|
|
199
246
|
|
|
200
247
|
def validate_table_fields(fields, file, failures, adapter_meta)
|
|
201
|
-
|
|
202
|
-
|
|
248
|
+
# handlers/table.rb:62 returns early, so a fieldless table deploys fine.
|
|
249
|
+
if fields.nil? || (fields.is_a?(Array) && fields.empty?)
|
|
250
|
+
semantic_warnings << {file: file, message: "Table has no fields, so it contributes nothing to the model yet"}
|
|
203
251
|
return
|
|
204
252
|
end
|
|
205
253
|
|
|
206
|
-
|
|
207
|
-
failures << {file: file, message: "'fields' must
|
|
254
|
+
unless fields.is_a?(Array)
|
|
255
|
+
failures << {file: file, message: "'fields' must be an array"}
|
|
208
256
|
return
|
|
209
257
|
end
|
|
210
258
|
|
|
@@ -221,6 +269,8 @@ module Strata
|
|
|
221
269
|
def validate_field_definition(field, file, idx, failures, adapter_meta)
|
|
222
270
|
field_name = field["name"] || "index #{idx}"
|
|
223
271
|
|
|
272
|
+
# handlers/table.rb:119 drops the rest, so on an existing field
|
|
273
|
+
# `datatype:` for `data_type:` silently leaves the old value.
|
|
224
274
|
unknown_keys = field.keys.map(&:to_s) - ALLOWED_FIELD_KEYS
|
|
225
275
|
if unknown_keys.any?
|
|
226
276
|
failures << {
|
|
@@ -234,6 +284,7 @@ module Strata
|
|
|
234
284
|
|
|
235
285
|
return unless field.key?("format")
|
|
236
286
|
|
|
287
|
+
# Normalized to nil and skipped (formattable.rb:10), so it never appears.
|
|
237
288
|
validate_field_format(field_name, field["format"], file, failures)
|
|
238
289
|
end
|
|
239
290
|
|
|
@@ -273,31 +324,40 @@ module Strata
|
|
|
273
324
|
failures << {file: file, message: "Field '#{field_name}' expression must include non-empty 'sql'"}
|
|
274
325
|
end
|
|
275
326
|
|
|
327
|
+
# ActiveModel::Type::Boolean casts everything outside FALSE_VALUES to
|
|
328
|
+
# true, so a quoted "no" stores the opposite. Bare `no` is fine.
|
|
276
329
|
EXPRESSION_BOOLEAN_KEYS.each do |key|
|
|
277
330
|
next unless expression.key?(key) || expression.key?(key.to_sym)
|
|
278
331
|
|
|
279
|
-
value = expression[key]
|
|
332
|
+
value = expression.key?(key) ? expression[key] : expression[key.to_sym]
|
|
280
333
|
next if value == true || value == false
|
|
281
334
|
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
message
|
|
285
|
-
|
|
335
|
+
message = "Field '#{field_name}' expression.#{key} is #{value.inspect}, not true or false"
|
|
336
|
+
if INVERTS_TO_TRUE.include?(value)
|
|
337
|
+
message += " — the server stores it as true; write it unquoted"
|
|
338
|
+
end
|
|
339
|
+
|
|
340
|
+
failures << {file: file, message: message}
|
|
286
341
|
end
|
|
287
342
|
end
|
|
288
343
|
|
|
289
|
-
|
|
344
|
+
# Reads as false, casts to true -- absent from FALSE_VALUES.
|
|
345
|
+
INVERTS_TO_TRUE = ["no", "No", "NO", "n", "N", "nil", "null", "none"].freeze
|
|
346
|
+
|
|
347
|
+
def validate_field_format(field_name, format_value, file, problems)
|
|
290
348
|
case format_value
|
|
349
|
+
when nil
|
|
350
|
+
problems << {file: file, message: "Field '#{field_name}' format must not be empty"}
|
|
291
351
|
when String
|
|
292
352
|
shortcut = format_value.strip
|
|
293
353
|
if shortcut.empty?
|
|
294
|
-
|
|
354
|
+
problems << {file: file, message: "Field '#{field_name}' format must not be empty"}
|
|
295
355
|
return
|
|
296
356
|
end
|
|
297
357
|
|
|
298
358
|
type = shortcut.split(":", 2).first
|
|
299
359
|
unless VALID_FORMAT_TYPES.include?(type)
|
|
300
|
-
|
|
360
|
+
problems << {
|
|
301
361
|
file: file,
|
|
302
362
|
message: "Field '#{field_name}' format shortcut has invalid type '#{type}' (expected one of: #{VALID_FORMAT_TYPES.join(", ")})"
|
|
303
363
|
}
|
|
@@ -305,15 +365,15 @@ module Strata
|
|
|
305
365
|
when Hash
|
|
306
366
|
type = format_value["type"] || format_value[:type]
|
|
307
367
|
if type.nil? || type.to_s.strip.empty?
|
|
308
|
-
|
|
368
|
+
problems << {file: file, message: "Field '#{field_name}' format hash must include type"}
|
|
309
369
|
elsif !VALID_FORMAT_TYPES.include?(type.to_s)
|
|
310
|
-
|
|
370
|
+
problems << {
|
|
311
371
|
file: file,
|
|
312
372
|
message: "Field '#{field_name}' format hash has invalid type '#{type}' (expected one of: #{VALID_FORMAT_TYPES.join(", ")})"
|
|
313
373
|
}
|
|
314
374
|
end
|
|
315
375
|
else
|
|
316
|
-
|
|
376
|
+
problems << {
|
|
317
377
|
file: file,
|
|
318
378
|
message: "Field '#{field_name}' format must be a string shortcut or a hash with type"
|
|
319
379
|
}
|
|
@@ -340,16 +400,19 @@ module Strata
|
|
|
340
400
|
validate_cardinality(name, definition["cardinality"], file, failures) if definition.key?("cardinality")
|
|
341
401
|
end
|
|
342
402
|
|
|
403
|
+
# relationships.rb:122 normalizes before the enum sees it:
|
|
404
|
+
# `join_def["cardinality"]&.downcase&.strip`.
|
|
343
405
|
def validate_cardinality(name, cardinality, file, failures)
|
|
344
|
-
return if RELATIONSHIP_CARDINALITIES.include?(cardinality)
|
|
406
|
+
return if RELATIONSHIP_CARDINALITIES.include?(cardinality.to_s.downcase.strip)
|
|
345
407
|
|
|
346
408
|
failures << {file: file, message: "Relationship '#{name}' has invalid cardinality '#{cardinality}'"}
|
|
347
409
|
end
|
|
348
410
|
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
411
|
+
# +unreachable+ takes the results: `connections` passes failures, `all`
|
|
412
|
+
# passes warnings -- the server never requires us to reach the warehouse.
|
|
413
|
+
def audit_connections(unreachable = nil)
|
|
352
414
|
failures = []
|
|
415
|
+
unreachable ||= failures
|
|
353
416
|
|
|
354
417
|
unless File.exist?("datasources.yml")
|
|
355
418
|
failures << {file: "datasources.yml", message: "File not found"}
|
|
@@ -362,11 +425,15 @@ module Strata
|
|
|
362
425
|
return failures
|
|
363
426
|
end
|
|
364
427
|
|
|
365
|
-
datasources.
|
|
428
|
+
datasources.each do |ds_key, config|
|
|
429
|
+
# Skip before create_adapter: it would install the adapter's driver gem to
|
|
430
|
+
# open a file that isn't here. warn_unverifiable_datasources names these.
|
|
431
|
+
next if local_file_missing?(config)
|
|
432
|
+
|
|
366
433
|
adapter = create_adapter(ds_key)
|
|
367
434
|
adapter.test_connection(raise_exception: true)
|
|
368
435
|
rescue => e
|
|
369
|
-
|
|
436
|
+
unreachable << {
|
|
370
437
|
file: "datasources.yml",
|
|
371
438
|
message: "Connection failed for '#{ds_key}': #{e.message}"
|
|
372
439
|
}
|
|
@@ -189,6 +189,12 @@ module Strata
|
|
|
189
189
|
end
|
|
190
190
|
rescue => e
|
|
191
191
|
say "\t!! Failed to connect: \n\t#{e.message}", :red
|
|
192
|
+
# datasources[] rather than ds_config: an unknown key is one of the ways we
|
|
193
|
+
# get here, and ds_config raises on it.
|
|
194
|
+
if local_file_missing?(datasources[ds_key])
|
|
195
|
+
say "\tThat path is not on this machine. If it lives inside the Strata container " \
|
|
196
|
+
"(like the sample's /data/tpcds.duckdb), the server opens it at deploy and this is expected.", :yellow
|
|
197
|
+
end
|
|
192
198
|
end
|
|
193
199
|
|
|
194
200
|
desc "tables DS_KEY", "List tables from DS_KEY datasource"
|
|
@@ -149,36 +149,7 @@ module Strata
|
|
|
149
149
|
end
|
|
150
150
|
|
|
151
151
|
def ensure_server(config)
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
config["server"] = collect_server_interactively
|
|
155
|
-
end
|
|
156
|
-
|
|
157
|
-
def collect_server_interactively
|
|
158
|
-
say "\nServer URL not found (not in project.yml)", :yellow
|
|
159
|
-
say "You can add it to project.yml or enter it now here\n", :cyan
|
|
160
|
-
|
|
161
|
-
server = prompt.ask("Enter Strata server URL:") do |q|
|
|
162
|
-
q.required true
|
|
163
|
-
q.validate(/\S+/, "Server URL cannot be empty")
|
|
164
|
-
end
|
|
165
|
-
|
|
166
|
-
server = server.strip.delete_suffix("/") # normalize: remove trailing slash
|
|
167
|
-
|
|
168
|
-
with_spinner("Saving server URL to project.yml") do
|
|
169
|
-
save_server_to_project_yml(server)
|
|
170
|
-
end
|
|
171
|
-
|
|
172
|
-
server
|
|
173
|
-
end
|
|
174
|
-
|
|
175
|
-
def save_server_to_project_yml(server)
|
|
176
|
-
Helpers::ProjectHelper.persist_server_to_project_yml(server, project_yml_path: File.join(project_path, Configuration::PROJECT_CONFIG_FILE))
|
|
177
|
-
CLI.config.reload!
|
|
178
|
-
return unless Utils::Git.git_repo?
|
|
179
|
-
|
|
180
|
-
Utils::Git.commit_file(Configuration::PROJECT_CONFIG_FILE, "[Strata-CLI] Add server URL to project.yml",
|
|
181
|
-
project_path)
|
|
152
|
+
config["server"] = Configuration.resolve_server(config["server"])
|
|
182
153
|
end
|
|
183
154
|
|
|
184
155
|
def ensure_git_url_populated
|