strata-cli 0.1.20 → 0.1.22

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -13,6 +13,7 @@ module Strata
13
13
  #
14
14
  # Server-only checks (universe/path integrity, physical table existence
15
15
  # in the warehouse, licenses) are deliberately not replicated here.
16
+
16
17
  module SemanticAuditChecks
17
18
  # deployer/handlers/table.rb dispatches on the literal string
18
19
  # "dimension"; any other value silently becomes a measure and a
@@ -24,7 +25,7 @@ module Strata
24
25
  DISPLAY_TYPES = %w[default html url email phone_number image].freeze
25
26
  DATE_GRAINS = %w[raw millisecond second minute hour day month week quarter year].freeze
26
27
  NUMERIC_DATA_TYPES = %w[integer decimal bigint].freeze
27
- # semantic/measure.rb + measure/excludable.rb + measure/includable.rb
28
+ # semantic/measure.rb + semantic/measure/excludable.rb + semantic/measure/includable.rb
28
29
  SNAPSHOT_TYPES = %w[beginning ending].freeze
29
30
  EXCLUSION_TYPES = %w[exclude exclude_all_except exclude_all].freeze
30
31
  EXCLUSION_ENTITY_TYPES = %w[dimension table universe].freeze
@@ -34,8 +35,8 @@ module Strata
34
35
  DYNAMIC_DATE = /(\d{1,3})([hdwmqy])/i
35
36
  # semantic/join_def.rb enums — note: no many_to_many.
36
37
  JOIN_TYPES = %w[inner left right].freeze
37
- # semantic/expression.rb:17 — the server accepts any word(...) call.
38
- MEASURE_SQL_FORMAT = /\A.*\w+\(.*\).*\z/i
38
+ # semantic/expression.rb:17 — any word(...) call. /m so a block scalar or a measure spanning lines still matches.
39
+ MEASURE_SQL_FORMAT = /\A.*\w+\(.*\).*\z/im
39
40
 
40
41
  private
41
42
 
@@ -65,9 +66,11 @@ module Strata
65
66
  next if name.empty?
66
67
 
67
68
  if seen_names[name]
69
+ # handlers/datasource.rb:18-20 merges them onto one row, so tables
70
+ # silently repoint at the other warehouse.
68
71
  failures << {
69
72
  file: "datasources.yml",
70
- message: "Datasource '#{key}' duplicates the name '#{config["name"]}' used by '#{seen_names[name]}'"
73
+ message: "Datasource '#{key}' duplicates the name '#{config["name"]}' used by '#{seen_names[name]}' (on deploy they merge into one datasource)"
71
74
  }
72
75
  else
73
76
  seen_names[name] = key
@@ -82,6 +85,7 @@ module Strata
82
85
  def validate_field_semantics(field, file, failures, meta)
83
86
  name = field["name"]
84
87
 
88
+ # handlers/table.rb:113 turns anything but "dimension" into a Measure.
85
89
  unless FIELD_TYPES.include?(field["type"].to_s.downcase.strip)
86
90
  failures << {
87
91
  file: file,
@@ -90,10 +94,29 @@ module Strata
90
94
  end
91
95
 
92
96
  validate_field_enums(name, field, file, failures)
93
- validate_measure_rules(name, field, file, failures)
97
+ if field["type"].to_s.downcase.strip == "dimension"
98
+ validate_no_measure_keys(name, field, file, failures)
99
+ else
100
+ validate_measure_rules(name, field, file, failures)
101
+ end
94
102
  validate_expression_semantics(name, field, file, failures, meta)
95
103
  end
96
104
 
105
+ # Written to a dimension by handlers/table.rb:119 but untyped, so
106
+ # `snapshot: ending` stores 0 and reads back as `beginning`.
107
+ MEASURE_ONLY_KEYS = %w[snapshot exclusion_type exclusions inclusions].freeze
108
+
109
+ def validate_no_measure_keys(name, field, file, failures)
110
+ present = MEASURE_ONLY_KEYS.select { field.key?(it) }
111
+ return if present.empty?
112
+
113
+ failures << {
114
+ file: file,
115
+ message: "Field '#{name}' is a dimension, so #{present.sort.join(", ")} #{(present.size == 1) ? "does" : "do"} not apply " \
116
+ "(#{(present.size == 1) ? "it is" : "they are"} measure-only; the server stores the value untyped and never reads it)"
117
+ }
118
+ end
119
+
97
120
  def validate_field_enums(name, field, file, failures)
98
121
  data_type = field["data_type"].to_s.strip
99
122
  if data_type.empty?
@@ -105,7 +128,8 @@ module Strata
105
128
  }
106
129
  end
107
130
 
108
- if field.key?("display_type") && !DISPLAY_TYPES.include?(field["display_type"].to_s)
131
+ # field.rb:16 is a nullable enum: an empty key is an absent one.
132
+ if !field["display_type"].to_s.strip.empty? && !DISPLAY_TYPES.include?(field["display_type"].to_s)
109
133
  failures << {
110
134
  file: file,
111
135
  message: "Field '#{name}' has invalid display_type '#{field["display_type"]}' (expected one of: #{DISPLAY_TYPES.join(", ")})"
@@ -119,7 +143,8 @@ module Strata
119
143
  end
120
144
 
121
145
  def validate_measure_rules(name, field, file, failures)
122
- if field.key?("snapshot") && !SNAPSHOT_TYPES.include?(field["snapshot"].to_s)
146
+ # measure.rb:6 is `enum :snapshot, [...], validate: {allow_nil: true}`.
147
+ if !field["snapshot"].to_s.strip.empty? && !SNAPSHOT_TYPES.include?(field["snapshot"].to_s)
123
148
  failures << {
124
149
  file: file,
125
150
  message: "Field '#{name}' has invalid snapshot '#{field["snapshot"]}' (expected beginning or ending)"
@@ -139,6 +164,16 @@ module Strata
139
164
  failures << {file: file, message: "Field '#{name}' exclusions should be empty when exclusion_type is exclude_all"}
140
165
  end
141
166
 
167
+ # excludable.rb:9,12 gates the rules on exclusion_type, so without it they
168
+ # are stored and never applied.
169
+ if exclusions.any? && exclusion_type.to_s.strip.empty?
170
+ failures << {
171
+ file: file,
172
+ message: "Field '#{name}' has exclusions but no 'exclusion_type', so the server stores the rules and never applies them"
173
+ }
174
+ exclusions = []
175
+ end
176
+
142
177
  exclusions.each_with_index do |rule, idx|
143
178
  unless rule.is_a?(Hash)
144
179
  failures << {file: file, message: "Field '#{name}' exclusion at index #{idx} must be a mapping"}
@@ -181,14 +216,14 @@ module Strata
181
216
  end
182
217
  end
183
218
 
184
- # ---- expressions (semantic/expression.rb, expression/references.rb) ----
219
+ # ---- expressions (semantic/expression.rb, semantic/expression/references.rb) ----
185
220
 
186
221
  def validate_expression_semantics(name, field, file, failures, meta)
187
222
  sql = expression_sql(field["expression"])
188
223
  return unless sql
189
224
 
190
225
  begin
191
- parsed = Sql::ExpressionParser.new(sql, reserved: meta[:reserved], aggregates: meta[:aggregates])
226
+ parsed = Sql::ExpressionParser.new(sql, reserved: meta[:reserved], aggregates: meta[:aggregates], quote: meta[:quote])
192
227
  rescue => e
193
228
  failures << {file: file, message: "Field '#{name}': expression could not be parsed: #{e.message}"}
194
229
  return
@@ -236,7 +271,7 @@ module Strata
236
271
  if types.nil?
237
272
  failures << {file: file, message: "Field '#{name}': [#{ref}] field was not found in the project"}
238
273
  elsif types.size > 1
239
- # references.rb:97
274
+ # references.rb:98
240
275
  failures << {
241
276
  file: file,
242
277
  message: "Field '#{name}': [#{ref}] matched multiple fields. Use @m or @d to specify the field further."
@@ -246,7 +281,7 @@ module Strata
246
281
  end
247
282
  end
248
283
 
249
- # references.rb:127-131
284
+ # references.rb:126-132
250
285
  if dimension && measure_refs.any?
251
286
  failures << {
252
287
  file: file,
@@ -286,7 +321,7 @@ module Strata
286
321
  return
287
322
  end
288
323
 
289
- # deployer/handlers/table.rb:204 + table/snappable.rb:40-43
324
+ # deployer/handlers/table.rb:204 + table/snappable.rb:38-43
290
325
  unless (project_semantic_index[:fields][snap_name] || []).include?("dimension")
291
326
  failures << {file: file, message: "Snapshot dimension '#{content["snapshot"]}' was not found in the project"}
292
327
  return
@@ -294,6 +329,8 @@ module Strata
294
329
 
295
330
  data_type = project_semantic_index[:dimension_data_types][snap_name].to_s
296
331
  unless %w[date date_time].include?(data_type)
332
+ # snappable.rb:38-43, which skip_reference_validation keeps off the
333
+ # deploy path -- snapshot_visitor then returns a wrong number silently.
297
334
  failures << {
298
335
  file: file,
299
336
  message: "Snapshot date '#{content["snapshot"]}' must be a date or date_time dimension (got '#{data_type}')"
@@ -388,11 +425,13 @@ module Strata
388
425
 
389
426
  sql = definition["sql"].to_s.downcase
390
427
  # join_def.rb:70-77
391
- if !sql.empty? && !(sql.match?(/left\./) && sql.match?(/right\./))
428
+ if !sql.empty? && !(sql.include?("left.") && sql.include?("right."))
392
429
  failures << {file: file, message: "Relationship '#{key}' join sql should be of the format left.column = right.column"}
393
430
  end
394
431
 
395
- if definition.key?("join") && !JOIN_TYPES.include?(definition["join"].to_s)
432
+ # relationships.rb:121 downcases and defaults to "inner" first.
433
+ join = definition["join"].to_s.downcase.strip
434
+ if !join.empty? && !JOIN_TYPES.include?(join)
396
435
  failures << {
397
436
  file: file,
398
437
  message: "Relationship '#{key}' has invalid join '#{definition["join"]}' (expected one of: #{JOIN_TYPES.join(", ")})"
@@ -404,19 +443,19 @@ module Strata
404
443
  end
405
444
 
406
445
  def validate_relationship_tables(key, definition, ds_key, file, failures)
407
- tables = project_semantic_index[:tables]
446
+ table_datasources = project_semantic_index[:table_datasources]
408
447
  left = definition["left"].to_s.downcase.strip
409
448
  right = definition["right"].to_s.downcase.strip
410
449
 
411
450
  [["left", left], ["right", right]].each do |side, table_name|
412
451
  next if table_name.empty?
413
452
 
414
- table = tables[table_name]
415
- if table.nil?
453
+ declared_in = table_datasources[table_name]
454
+ if declared_in.nil?
416
455
  # deployer/handlers/relationships.rb:106-107
417
456
  failures << {file: file, message: "Relationship '#{key}': #{side} table '#{definition[side]}' was not found in the project"}
418
- elsif ds_key && (table_ds = resolve_datasource_value(table["datasource"])) && table_ds != ds_key
419
- # join_def.rb:65-67
457
+ elsif ds_key && !declared_in.include?(ds_key)
458
+ # join_def.rb:65-67 — the name is taken in some other datasource only.
420
459
  failures << {
421
460
  file: file,
422
461
  message: "Relationship '#{key}': #{side} table '#{definition[side]}' is not in datasource '#{ds_key}' (tables can only join within one datasource)"
@@ -427,14 +466,18 @@ module Strata
427
466
  return if left.empty? || right.empty?
428
467
 
429
468
  # join_def.rb:79-83 — joins are bi-directional, so either order collides.
430
- pair = [left, right].sort
469
+ # Scoped by datasource: the pair identifies table records.
470
+ pair = [ds_key, *[left, right].sort]
431
471
  if (existing = seen_join_pairs[pair])
472
+ # relationships.rb:113-115 raises within a file; across files
473
+ # first_or_initialize lets glob order decide. Non-deterministic either way.
432
474
  failures << {
433
475
  file: file,
434
- message: "Relationship '#{key}': a join between '#{definition["left"]}' and '#{definition["right"]}' already exists (#{existing})"
476
+ message: "Relationship '#{key}': a join between '#{definition["left"]}' and '#{definition["right"]}' " \
477
+ "already exists (#{existing[:file]}: #{existing[:key]})"
435
478
  }
436
479
  else
437
- seen_join_pairs[pair] = "#{file}: #{key}"
480
+ seen_join_pairs[pair] = {file: file, key: key}
438
481
  end
439
482
  end
440
483
 
@@ -442,19 +485,26 @@ module Strata
442
485
  @seen_join_pairs ||= {}
443
486
  end
444
487
 
488
+ # Reported without failing the audit.
489
+ def semantic_warnings
490
+ @semantic_warnings ||= []
491
+ end
492
+
445
493
  # ---- shared infrastructure ----
446
494
 
447
495
  # Name index of every table and field declared across the project's
448
496
  # tbl.*.yml files, keyed by lowercased name (the server resolves
449
497
  # references by lower(name), and the expression parser downcases its
450
498
  # input, so lowercase keys line up on both sides).
499
+ # Fields stay keyed by name alone (uniqueness scope: [:type, :branch_id]);
500
+ # tables are per-datasource, so that index maps a name to its datasources.
451
501
  def project_semantic_index
452
502
  @project_semantic_index ||= begin
453
503
  fields = {}
454
504
  dimension_data_types = {}
455
- tables = {}
505
+ table_datasources = {}
456
506
 
457
- Dir.glob("models/**/*.yml").each do |file|
507
+ Dir.glob("models/**/*.{yml,yaml}").each do |file|
458
508
  next unless File.basename(file).start_with?("tbl.")
459
509
 
460
510
  content = begin
@@ -466,7 +516,9 @@ module Strata
466
516
 
467
517
  table_name = content["name"].to_s.downcase.strip
468
518
  unless table_name.empty?
469
- tables[table_name] = {"datasource" => content["datasource"], "file" => file}
519
+ # Unique per datasource (semantic/table.rb:32), so one name can
520
+ # belong to several.
521
+ (table_datasources[table_name] ||= []) << resolve_datasource_value(content["datasource"])
470
522
  end
471
523
 
472
524
  Array(content["fields"]).each do |f|
@@ -481,7 +533,7 @@ module Strata
481
533
  end
482
534
  end
483
535
 
484
- {fields: fields, tables: tables, dimension_data_types: dimension_data_types}
536
+ {fields: fields, table_datasources: table_datasources, dimension_data_types: dimension_data_types}
485
537
  end
486
538
  end
487
539
 
@@ -492,14 +544,20 @@ module Strata
492
544
  def adapter_meta(ds_value)
493
545
  @adapter_meta ||= {}
494
546
  key = resolve_datasource_value(ds_value)
495
- config = (key && (datasources[key] || datasources[key.to_s.downcase.strip])) || {}
547
+ config = (key && datasources[key]) || {}
496
548
  sym = config["adapter"].to_s.downcase.strip.to_sym
497
549
 
498
550
  @adapter_meta[sym] ||= if DWH.adapter?(sym)
499
551
  klass = DWH.get_adapter(sym)
500
- {reserved: klass.reserved_keywords, aggregates: klass.aggregate_functions}
552
+ {
553
+ reserved: klass.reserved_keywords,
554
+ aggregates: klass.aggregate_functions,
555
+ # references.rb:5-8 passes the adapter's own identifier quote, so a
556
+ # backtick warehouse reads `order` as a column.
557
+ quote: klass.adapter_settings[:quote]
558
+ }
501
559
  else
502
- {reserved: DWH.reserved_keywords, aggregates: DWH.aggregate_functions}
560
+ {reserved: DWH.reserved_keywords, aggregates: DWH.aggregate_functions, quote: Sql::ExpressionParser::ANSI_QUOTE}
503
561
  end
504
562
  end
505
563
  end
@@ -30,21 +30,19 @@ module Strata
30
30
  say VERSION
31
31
  end
32
32
 
33
- desc "init [PROJECT_NAME]", "Initializes a new Strata project. PROJECT_NAME is optional when using --source."
33
+ desc "init PROJECT_NAME", "Initializes a new Strata project."
34
34
  long_desc_from_file "init"
35
35
  option :datasource, aliases: ["d"], type: :string, desc: "One of the supported data warehouse adapters.",
36
36
  repeatable: true
37
- option :source, aliases: ["s"], type: :string, desc: "URL of existing project"
38
- option :api_key, aliases: ["a"], type: :string, desc: "Api Key. Required if initializing existing project."
37
+ option :server, aliases: ["s"], type: :string, desc: "Strata server URL. Defaults to #{Configuration::DEFAULT_SERVER}."
38
+ option :api_key, aliases: ["a"], type: :string, desc: "Api Key. Optional — the first deploy signs you in via browser."
39
39
  option :verbose, aliases: ["v"], type: :boolean, default: false,
40
40
  desc: "Show detailed init output (file-by-file actions)."
41
41
 
42
42
  def init(project_name = nil)
43
- unless project_name || options[:source]
44
- raise Strata::CommandError, "PROJECT_NAME is required when not using --source option."
45
- end
43
+ raise Strata::CommandError, "PROJECT_NAME is required." unless project_name
46
44
 
47
- print_status(:created, (project_name || "project from source").to_s, type: :info)
45
+ print_status(:created, project_name.to_s, type: :info)
48
46
  invoke Generators::Project, [project_name], options
49
47
  end
50
48
 
@@ -24,7 +24,7 @@ module Strata
24
24
  include SemanticAuditChecks
25
25
  include AgentMode
26
26
 
27
- REQUIRED_KEYS_FOR_TABLE_MODEL = %w[name physical_name fields datasource].freeze
27
+ REQUIRED_KEYS_FOR_TABLE_MODEL = %w[name physical_name datasource].freeze
28
28
  REQUIRED_KEYS_FOR_RELATIONSHIP_MODEL = ["datasource"].freeze
29
29
  REQUIRED_KEYS_FOR_RELATIONSHIP_DEFINITION = %w[left right sql cardinality].freeze
30
30
  RELATIONSHIP_CARDINALITIES = %w[one_to_one one_to_many many_to_one].freeze
@@ -35,7 +35,15 @@ module Strata
35
35
  exclusions inclusions extended_blend_group synonyms expression tags
36
36
  ].freeze
37
37
  ALLOWED_EXPRESSION_KEYS = %w[sql array lookup primary_key].freeze
38
+ # What the deployer reads from a tbl file (handlers/table.rb:4, :62, :71,
39
+ # :152, :202), plus the CLI's own imports.
40
+ ALLOWED_TABLE_KEYS = %w[
41
+ name description physical_name cost metadata tags
42
+ datasource fields snapshot partition partitions imports
43
+ ].freeze
38
44
  STRAY_SCRIPT_EXTENSIONS = %w[.rb .py .sh .ipynb].freeze
45
+ # deploy_job.rb:121,162 globs models as {yml,yaml}, so both are models.
46
+ MODEL_EXTENSIONS = %w[.yml .yaml].freeze
39
47
  EXPRESSION_BOOLEAN_KEYS = %w[array lookup primary_key].freeze
40
48
 
41
49
  # Set default command so `strata audit` still works as `strata audit all`
@@ -46,8 +54,10 @@ module Strata
46
54
  results = {}
47
55
  results[:yaml] = run_check("Checking YAML syntax") { audit_yaml_syntax }
48
56
  results[:models] = run_check("Checking model definitions") { audit_models }
49
- results[:connections] = run_check("Checking data source connections") { audit_connections }
57
+ results[:connections] = run_check("Checking data source connections") { audit_connections(semantic_warnings) }
50
58
 
59
+ warn_semantic_warnings
60
+ warn_unverifiable_datasources
51
61
  warn_stray_files
52
62
  report_results(results)
53
63
  end
@@ -61,12 +71,15 @@ module Strata
61
71
  desc "models", "Check model definitions and schema structure"
62
72
  def models
63
73
  results = {models: run_check("Checking model definitions") { audit_models }}
74
+ warn_semantic_warnings
64
75
  report_results(results)
65
76
  end
66
77
 
67
78
  desc "connections", "Check data source connections"
68
79
  def connections
69
80
  results = {connections: run_check("Checking data source connections") { audit_connections }}
81
+ warn_semantic_warnings
82
+ warn_unverifiable_datasources
70
83
  report_results(results)
71
84
  end
72
85
 
@@ -84,7 +97,7 @@ module Strata
84
97
  # A Strata project holds YAML, not scripts. Agents sometimes leave scratch
85
98
  # validators behind; warn, don't fail — a helper script is the user's call.
86
99
  def warn_stray_files
87
- strays = Dir.glob("models/**/*").select { File.file?(it) && File.extname(it) != ".yml" }
100
+ strays = Dir.glob("models/**/*").select { File.file?(it) && !MODEL_EXTENSIONS.include?(File.extname(it)) }
88
101
  strays += Dir.glob("*").select { File.file?(it) && STRAY_SCRIPT_EXTENSIONS.include?(File.extname(it)) }
89
102
  return if strays.empty?
90
103
 
@@ -92,6 +105,25 @@ module Strata
92
105
  strays.sort.each { print_warning(" #{it}") }
93
106
  end
94
107
 
108
+ # Problems the server tolerates: shown, but never failed.
109
+ def warn_semantic_warnings
110
+ return if semantic_warnings.empty?
111
+
112
+ print_warning("\n Accepted by the server, but worth a look:")
113
+ semantic_warnings.each { print_warning(" #{it[:file]}: #{it[:message]}") }
114
+ end
115
+
116
+ # Pointing a datasource at a path the server owns is a normal setup — the sample
117
+ # project's /data/tpcds.duckdb ships inside the Strata image — so say which
118
+ # connections went unchecked instead of failing a correct project.
119
+ def warn_unverifiable_datasources
120
+ skipped = datasources.select { |_key, config| local_file_missing?(config) }
121
+ return if skipped.empty?
122
+
123
+ print_warning("\n Connection not checked — file not on this machine (the server opens it at deploy):")
124
+ skipped.each { |key, config| print_warning(" #{key}: #{config["file"]}") }
125
+ end
126
+
95
127
  def run_check(message)
96
128
  failures = []
97
129
  with_spinner(message) do
@@ -108,7 +140,7 @@ module Strata
108
140
  end
109
141
 
110
142
  def audit_yaml_syntax
111
- files = Dir.glob("models/**/*.yml") + ["datasources.yml"].select { |f| File.exist?(f) }
143
+ files = Dir.glob("models/**/*.{yml,yaml}") + ["datasources.yml"].select { |f| File.exist?(f) }
112
144
 
113
145
  failures = []
114
146
  files.each do |file|
@@ -123,7 +155,7 @@ module Strata
123
155
 
124
156
  def audit_models
125
157
  failures = audit_datasource_definitions
126
- Dir.glob("models/**/*.yml").each do |file|
158
+ Dir.glob("models/**/*.{yml,yaml}").each do |file|
127
159
  audit_imports(file, failures)
128
160
  audit_model_file(file, failures)
129
161
  end
@@ -160,19 +192,34 @@ module Strata
160
192
  end
161
193
  end
162
194
 
195
+ # handlers/table.rb:54 drops it, so an unquoted `yes:` does nothing.
163
196
  def validate_structure(content, file, failures)
164
197
  content.each_key do |k|
165
- failures << {file: file, message: "Top level key #{k} is not a string"} unless k.is_a?(String)
198
+ next if k.is_a?(String)
199
+
200
+ failures << {file: file, message: "Top level key #{k} is not a string"}
166
201
  end
167
202
  end
168
203
 
169
204
  def validate_table_model(content, file, failures)
170
205
  validate_required_keys(content, file, REQUIRED_KEYS_FOR_TABLE_MODEL, failures)
206
+ validate_table_keys(content, file, failures)
171
207
  validate_datasource_reference(content, file, failures)
172
208
  validate_table_fields(content["fields"], file, failures, adapter_meta(content["datasource"]))
173
209
  validate_table_semantics(content, file, failures)
174
210
  end
175
211
 
212
+ # Same slice: `snapshot_date:` deploys a table with no snapshot date.
213
+ def validate_table_keys(content, file, failures)
214
+ unknown_keys = content.keys.select { it.is_a?(String) }.map(&:to_s) - ALLOWED_TABLE_KEYS
215
+ return if unknown_keys.empty?
216
+
217
+ failures << {
218
+ file: file,
219
+ message: "Table has unknown top-level #{(unknown_keys.size == 1) ? "key" : "keys"}: #{unknown_keys.sort.join(", ")}"
220
+ }
221
+ end
222
+
176
223
  def validate_relationship_model(content, file, failures)
177
224
  validate_required_keys(content, file, REQUIRED_KEYS_FOR_RELATIONSHIP_MODEL, failures)
178
225
  validate_datasource_reference(content, file, failures)
@@ -198,13 +245,14 @@ module Strata
198
245
  end
199
246
 
200
247
  def validate_table_fields(fields, file, failures, adapter_meta)
201
- unless fields.is_a?(Array)
202
- failures << {file: file, message: "'fields' must be an array"}
248
+ # handlers/table.rb:62 returns early, so a fieldless table deploys fine.
249
+ if fields.nil? || (fields.is_a?(Array) && fields.empty?)
250
+ semantic_warnings << {file: file, message: "Table has no fields, so it contributes nothing to the model yet"}
203
251
  return
204
252
  end
205
253
 
206
- if fields.empty?
207
- failures << {file: file, message: "'fields' must not be empty"}
254
+ unless fields.is_a?(Array)
255
+ failures << {file: file, message: "'fields' must be an array"}
208
256
  return
209
257
  end
210
258
 
@@ -221,6 +269,8 @@ module Strata
221
269
  def validate_field_definition(field, file, idx, failures, adapter_meta)
222
270
  field_name = field["name"] || "index #{idx}"
223
271
 
272
+ # handlers/table.rb:119 drops the rest, so on an existing field
273
+ # `datatype:` for `data_type:` silently leaves the old value.
224
274
  unknown_keys = field.keys.map(&:to_s) - ALLOWED_FIELD_KEYS
225
275
  if unknown_keys.any?
226
276
  failures << {
@@ -234,6 +284,7 @@ module Strata
234
284
 
235
285
  return unless field.key?("format")
236
286
 
287
+ # Normalized to nil and skipped (formattable.rb:10), so it never appears.
237
288
  validate_field_format(field_name, field["format"], file, failures)
238
289
  end
239
290
 
@@ -273,31 +324,40 @@ module Strata
273
324
  failures << {file: file, message: "Field '#{field_name}' expression must include non-empty 'sql'"}
274
325
  end
275
326
 
327
+ # ActiveModel::Type::Boolean casts everything outside FALSE_VALUES to
328
+ # true, so a quoted "no" stores the opposite. Bare `no` is fine.
276
329
  EXPRESSION_BOOLEAN_KEYS.each do |key|
277
330
  next unless expression.key?(key) || expression.key?(key.to_sym)
278
331
 
279
- value = expression[key] || expression[key.to_sym]
332
+ value = expression.key?(key) ? expression[key] : expression[key.to_sym]
280
333
  next if value == true || value == false
281
334
 
282
- failures << {
283
- file: file,
284
- message: "Field '#{field_name}' expression.#{key} must be true or false"
285
- }
335
+ message = "Field '#{field_name}' expression.#{key} is #{value.inspect}, not true or false"
336
+ if INVERTS_TO_TRUE.include?(value)
337
+ message += " — the server stores it as true; write it unquoted"
338
+ end
339
+
340
+ failures << {file: file, message: message}
286
341
  end
287
342
  end
288
343
 
289
- def validate_field_format(field_name, format_value, file, failures)
344
+ # Reads as false, casts to true -- absent from FALSE_VALUES.
345
+ INVERTS_TO_TRUE = ["no", "No", "NO", "n", "N", "nil", "null", "none"].freeze
346
+
347
+ def validate_field_format(field_name, format_value, file, problems)
290
348
  case format_value
349
+ when nil
350
+ problems << {file: file, message: "Field '#{field_name}' format must not be empty"}
291
351
  when String
292
352
  shortcut = format_value.strip
293
353
  if shortcut.empty?
294
- failures << {file: file, message: "Field '#{field_name}' format must not be empty"}
354
+ problems << {file: file, message: "Field '#{field_name}' format must not be empty"}
295
355
  return
296
356
  end
297
357
 
298
358
  type = shortcut.split(":", 2).first
299
359
  unless VALID_FORMAT_TYPES.include?(type)
300
- failures << {
360
+ problems << {
301
361
  file: file,
302
362
  message: "Field '#{field_name}' format shortcut has invalid type '#{type}' (expected one of: #{VALID_FORMAT_TYPES.join(", ")})"
303
363
  }
@@ -305,15 +365,15 @@ module Strata
305
365
  when Hash
306
366
  type = format_value["type"] || format_value[:type]
307
367
  if type.nil? || type.to_s.strip.empty?
308
- failures << {file: file, message: "Field '#{field_name}' format hash must include type"}
368
+ problems << {file: file, message: "Field '#{field_name}' format hash must include type"}
309
369
  elsif !VALID_FORMAT_TYPES.include?(type.to_s)
310
- failures << {
370
+ problems << {
311
371
  file: file,
312
372
  message: "Field '#{field_name}' format hash has invalid type '#{type}' (expected one of: #{VALID_FORMAT_TYPES.join(", ")})"
313
373
  }
314
374
  end
315
375
  else
316
- failures << {
376
+ problems << {
317
377
  file: file,
318
378
  message: "Field '#{field_name}' format must be a string shortcut or a hash with type"
319
379
  }
@@ -340,16 +400,19 @@ module Strata
340
400
  validate_cardinality(name, definition["cardinality"], file, failures) if definition.key?("cardinality")
341
401
  end
342
402
 
403
+ # relationships.rb:122 normalizes before the enum sees it:
404
+ # `join_def["cardinality"]&.downcase&.strip`.
343
405
  def validate_cardinality(name, cardinality, file, failures)
344
- return if RELATIONSHIP_CARDINALITIES.include?(cardinality)
406
+ return if RELATIONSHIP_CARDINALITIES.include?(cardinality.to_s.downcase.strip)
345
407
 
346
408
  failures << {file: file, message: "Relationship '#{name}' has invalid cardinality '#{cardinality}'"}
347
409
  end
348
410
 
349
- def audit_connections
350
- return [] if CLI.config.get("git").to_s.include?("tpc-ds-retail-sample")
351
-
411
+ # +unreachable+ takes the results: `connections` passes failures, `all`
412
+ # passes warnings -- the server never requires us to reach the warehouse.
413
+ def audit_connections(unreachable = nil)
352
414
  failures = []
415
+ unreachable ||= failures
353
416
 
354
417
  unless File.exist?("datasources.yml")
355
418
  failures << {file: "datasources.yml", message: "File not found"}
@@ -362,11 +425,15 @@ module Strata
362
425
  return failures
363
426
  end
364
427
 
365
- datasources.each_key do |ds_key|
428
+ datasources.each do |ds_key, config|
429
+ # Skip before create_adapter: it would install the adapter's driver gem to
430
+ # open a file that isn't here. warn_unverifiable_datasources names these.
431
+ next if local_file_missing?(config)
432
+
366
433
  adapter = create_adapter(ds_key)
367
434
  adapter.test_connection(raise_exception: true)
368
435
  rescue => e
369
- failures << {
436
+ unreachable << {
370
437
  file: "datasources.yml",
371
438
  message: "Connection failed for '#{ds_key}': #{e.message}"
372
439
  }
@@ -189,6 +189,12 @@ module Strata
189
189
  end
190
190
  rescue => e
191
191
  say "\t!! Failed to connect: \n\t#{e.message}", :red
192
+ # datasources[] rather than ds_config: an unknown key is one of the ways we
193
+ # get here, and ds_config raises on it.
194
+ if local_file_missing?(datasources[ds_key])
195
+ say "\tThat path is not on this machine. If it lives inside the Strata container " \
196
+ "(like the sample's /data/tpcds.duckdb), the server opens it at deploy and this is expected.", :yellow
197
+ end
192
198
  end
193
199
 
194
200
  desc "tables DS_KEY", "List tables from DS_KEY datasource"
@@ -149,36 +149,7 @@ module Strata
149
149
  end
150
150
 
151
151
  def ensure_server(config)
152
- return if config["server"] && !config["server"].to_s.strip.empty?
153
-
154
- config["server"] = collect_server_interactively
155
- end
156
-
157
- def collect_server_interactively
158
- say "\nServer URL not found (not in project.yml)", :yellow
159
- say "You can add it to project.yml or enter it now here\n", :cyan
160
-
161
- server = prompt.ask("Enter Strata server URL:") do |q|
162
- q.required true
163
- q.validate(/\S+/, "Server URL cannot be empty")
164
- end
165
-
166
- server = server.strip.delete_suffix("/") # normalize: remove trailing slash
167
-
168
- with_spinner("Saving server URL to project.yml") do
169
- save_server_to_project_yml(server)
170
- end
171
-
172
- server
173
- end
174
-
175
- def save_server_to_project_yml(server)
176
- Helpers::ProjectHelper.persist_server_to_project_yml(server, project_yml_path: File.join(project_path, Configuration::PROJECT_CONFIG_FILE))
177
- CLI.config.reload!
178
- return unless Utils::Git.git_repo?
179
-
180
- Utils::Git.commit_file(Configuration::PROJECT_CONFIG_FILE, "[Strata-CLI] Add server URL to project.yml",
181
- project_path)
152
+ config["server"] = Configuration.resolve_server(config["server"])
182
153
  end
183
154
 
184
155
  def ensure_git_url_populated