manifold-cli 0.0.18 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/manifold/api/schema_manager.rb +277 -0
- data/lib/manifold/api/workspace.rb +11 -91
- data/lib/manifold/services/vector_service.rb +1 -1
- data/lib/manifold/templates/workspace_template.yml +15 -3
- data/lib/manifold/terraform/project_configuration.rb +1 -1
- data/lib/manifold/terraform/workspace_configuration.rb +59 -51
- data/lib/manifold/version.rb +1 -1
- metadata +3 -3
- data/lib/manifold/api/schema_generator.rb +0 -89
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c51fac99031686a4856753b93cd64ad3de0b6c304aefaa6f4c7b40db19c03b30
|
4
|
+
data.tar.gz: 907654d8f6f18061dd26d5a47299f541aefbb36657c5ef6267d041a581c53430
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ceb3039ef12f726c2c8c95ddb9378b945fb37ee1249040a75c910a9bb6a165534d47913e42508e1c5ba150187f514473de7aa0e2113f3e5e359f5661241bf3b8
|
7
|
+
data.tar.gz: de3a2ef465a7f7ed31f8e8a26972b42baed631c70b9fe877ce23033eda95b8c4f1dd0c5238a4113e8d09c4e760e6a8d1b9dbe979bcabd5df059075f9fec6efc0
|
@@ -0,0 +1,277 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Manifold
|
4
|
+
module API
|
5
|
+
# Handles schema generation and writing for Manifold tables
|
6
|
+
# rubocop:disable Metrics/ClassLength
|
7
|
+
class SchemaManager
|
8
|
+
def initialize(name, vectors, vector_service, manifold_yaml, logger)
|
9
|
+
@name = name
|
10
|
+
@vectors = vectors
|
11
|
+
@vector_service = vector_service
|
12
|
+
@manifold_yaml = manifold_yaml
|
13
|
+
@logger = logger
|
14
|
+
end
|
15
|
+
|
16
|
+
# Generates and writes schemas to the specified directory
|
17
|
+
def write_schemas(tables_directory)
|
18
|
+
tables_directory.mkpath
|
19
|
+
write_dimensions_schema(tables_directory)
|
20
|
+
write_manifold_schema(tables_directory)
|
21
|
+
write_metrics_schemas(tables_directory)
|
22
|
+
end
|
23
|
+
|
24
|
+
# Returns the dimensions schema structure
|
25
|
+
def dimensions_schema
|
26
|
+
[
|
27
|
+
{ "type" => "STRING", "name" => "id", "mode" => "REQUIRED" },
|
28
|
+
{ "type" => "RECORD", "name" => "dimensions", "mode" => "REQUIRED",
|
29
|
+
"fields" => dimensions_fields }
|
30
|
+
]
|
31
|
+
end
|
32
|
+
|
33
|
+
# Returns the manifold schema structure
|
34
|
+
def manifold_schema
|
35
|
+
[
|
36
|
+
{ "type" => "STRING", "name" => "id", "mode" => "REQUIRED" },
|
37
|
+
{ "type" => "TIMESTAMP", "name" => "timestamp", "mode" => "REQUIRED" },
|
38
|
+
{ "type" => "RECORD", "name" => "dimensions", "mode" => "REQUIRED",
|
39
|
+
"fields" => dimensions_fields },
|
40
|
+
{ "type" => "RECORD", "name" => "metrics", "mode" => "REQUIRED",
|
41
|
+
"fields" => metrics_fields }
|
42
|
+
]
|
43
|
+
end
|
44
|
+
|
45
|
+
private
|
46
|
+
|
47
|
+
def write_dimensions_schema(tables_directory)
|
48
|
+
dimensions_path = tables_directory.join("dimensions.json")
|
49
|
+
dimensions_path.write(dimensions_schema_json.concat("\n"))
|
50
|
+
end
|
51
|
+
|
52
|
+
def write_manifold_schema(tables_directory)
|
53
|
+
manifold_path = tables_directory.join("manifold.json")
|
54
|
+
manifold_path.write(manifold_schema_json.concat("\n"))
|
55
|
+
end
|
56
|
+
|
57
|
+
def write_metrics_schemas(tables_directory)
|
58
|
+
return unless @manifold_yaml["metrics"]
|
59
|
+
|
60
|
+
create_metrics_directory(tables_directory)
|
61
|
+
write_individual_metrics_schemas(tables_directory)
|
62
|
+
end
|
63
|
+
|
64
|
+
def create_metrics_directory(tables_directory)
|
65
|
+
metrics_directory = tables_directory.join("metrics")
|
66
|
+
metrics_directory.mkpath
|
67
|
+
end
|
68
|
+
|
69
|
+
def write_individual_metrics_schemas(tables_directory)
|
70
|
+
@manifold_yaml["metrics"].each do |group_name, group_config|
|
71
|
+
write_metrics_group_schema(tables_directory, group_name, group_config)
|
72
|
+
end
|
73
|
+
end
|
74
|
+
|
75
|
+
def write_metrics_group_schema(tables_directory, group_name, group_config)
|
76
|
+
metrics_table_path = tables_directory.join("metrics", "#{group_name}.json")
|
77
|
+
metrics_table_schema = metrics_table_schema(group_name, group_config)
|
78
|
+
metrics_table_path.write(JSON.pretty_generate(metrics_table_schema).concat("\n"))
|
79
|
+
@logger.info("Generated metrics table schema for '#{group_name}'.")
|
80
|
+
end
|
81
|
+
|
82
|
+
def metrics_table_schema(group_name, group_config)
|
83
|
+
[
|
84
|
+
{ "type" => "STRING", "name" => "id", "mode" => "REQUIRED" },
|
85
|
+
{ "type" => "TIMESTAMP", "name" => "timestamp", "mode" => "REQUIRED" },
|
86
|
+
{ "type" => "RECORD", "name" => "metrics", "mode" => "REQUIRED",
|
87
|
+
"fields" => [metrics_group_field(group_name, group_config)] }
|
88
|
+
]
|
89
|
+
end
|
90
|
+
|
91
|
+
def metrics_group_field(group_name, group_config)
|
92
|
+
{
|
93
|
+
"name" => group_name,
|
94
|
+
"type" => "RECORD",
|
95
|
+
"mode" => "NULLABLE",
|
96
|
+
"fields" => group_metrics_fields(group_config)
|
97
|
+
}
|
98
|
+
end
|
99
|
+
|
100
|
+
def dimensions_fields
|
101
|
+
@dimensions_fields ||= @vectors.filter_map do |vector|
|
102
|
+
@logger.info("Loading vector schema for '#{vector}'.")
|
103
|
+
@vector_service.load_vector_schema(vector)
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
107
|
+
def dimensions_schema_json
|
108
|
+
JSON.pretty_generate(dimensions_schema)
|
109
|
+
end
|
110
|
+
|
111
|
+
def manifold_schema_json
|
112
|
+
JSON.pretty_generate(manifold_schema)
|
113
|
+
end
|
114
|
+
|
115
|
+
def metrics_fields
|
116
|
+
return [] unless @manifold_yaml["metrics"]
|
117
|
+
|
118
|
+
@manifold_yaml["metrics"].map do |group_name, group_config|
|
119
|
+
{
|
120
|
+
"name" => group_name,
|
121
|
+
"type" => "RECORD",
|
122
|
+
"mode" => "NULLABLE",
|
123
|
+
"fields" => group_metrics_fields(group_config)
|
124
|
+
}
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
def group_metrics_fields(group_config)
|
129
|
+
return [] unless group_config["aggregations"]
|
130
|
+
|
131
|
+
# Generate condition fields
|
132
|
+
condition_fields = generate_condition_fields(get_conditions_list(group_config), group_config)
|
133
|
+
|
134
|
+
# Generate intersection fields between breakout groups
|
135
|
+
intersection_fields = generate_breakout_intersection_fields(group_config)
|
136
|
+
|
137
|
+
condition_fields + intersection_fields
|
138
|
+
end
|
139
|
+
|
140
|
+
def get_conditions_list(group_config)
|
141
|
+
return [] unless group_config["conditions"]
|
142
|
+
|
143
|
+
group_config["conditions"].keys
|
144
|
+
end
|
145
|
+
|
146
|
+
def create_metric_field(field_name, group_config)
|
147
|
+
{
|
148
|
+
"name" => field_name,
|
149
|
+
"type" => "RECORD",
|
150
|
+
"mode" => "NULLABLE",
|
151
|
+
"fields" => breakout_metrics_fields(group_config)
|
152
|
+
}
|
153
|
+
end
|
154
|
+
|
155
|
+
def generate_condition_fields(conditions, group_config)
|
156
|
+
conditions.map do |condition_name|
|
157
|
+
create_metric_field(condition_name, group_config)
|
158
|
+
end
|
159
|
+
end
|
160
|
+
|
161
|
+
def generate_breakout_intersection_fields(group_config)
|
162
|
+
return [] unless group_config["breakouts"]
|
163
|
+
return [] if group_config["breakouts"].keys.size <= 1
|
164
|
+
|
165
|
+
generate_all_breakout_combinations(group_config)
|
166
|
+
end
|
167
|
+
|
168
|
+
def generate_all_breakout_combinations(group_config)
|
169
|
+
all_intersection_fields = []
|
170
|
+
breakout_groups = group_config["breakouts"].keys
|
171
|
+
|
172
|
+
# Generate combinations of different sizes (2 to n breakout groups)
|
173
|
+
(2..breakout_groups.size).each do |combination_size|
|
174
|
+
add_combinations_of_size(combination_size, breakout_groups, group_config, all_intersection_fields)
|
175
|
+
end
|
176
|
+
|
177
|
+
all_intersection_fields
|
178
|
+
end
|
179
|
+
|
180
|
+
def add_combinations_of_size(size, breakout_groups, group_config, all_fields)
|
181
|
+
breakout_groups.combination(size).each do |breakout_combination|
|
182
|
+
fields = generate_intersection_fields_for_combination(group_config, breakout_combination)
|
183
|
+
all_fields.concat(fields)
|
184
|
+
end
|
185
|
+
end
|
186
|
+
|
187
|
+
def generate_intersection_fields_for_combination(group_config, breakout_combination)
|
188
|
+
# Get all conditions from the given breakout groups
|
189
|
+
condition_sets = breakout_combination.map do |breakout_group|
|
190
|
+
group_config["breakouts"][breakout_group]
|
191
|
+
end
|
192
|
+
|
193
|
+
# Generate all combinations of one condition from each breakout group
|
194
|
+
generate_all_condition_combinations(condition_sets, group_config)
|
195
|
+
end
|
196
|
+
|
197
|
+
def generate_all_condition_combinations(condition_sets, group_config)
|
198
|
+
# Start with first breakout group's conditions
|
199
|
+
combinations = condition_sets.first.map { |condition| [condition] }
|
200
|
+
|
201
|
+
# Extend combinations with remaining breakout groups
|
202
|
+
extended_combinations = extend_combinations_with_remaining_sets(combinations, condition_sets[1..])
|
203
|
+
|
204
|
+
# Convert combinations to field definitions
|
205
|
+
create_intersection_fields(extended_combinations, group_config)
|
206
|
+
end
|
207
|
+
|
208
|
+
def extend_combinations_with_remaining_sets(initial_combinations, remaining_sets)
|
209
|
+
combinations = initial_combinations
|
210
|
+
|
211
|
+
remaining_sets.each do |conditions|
|
212
|
+
combinations = extend_combinations_with_conditions(combinations, conditions)
|
213
|
+
end
|
214
|
+
|
215
|
+
combinations
|
216
|
+
end
|
217
|
+
|
218
|
+
def extend_combinations_with_conditions(existing_combinations, conditions)
|
219
|
+
new_combinations = []
|
220
|
+
|
221
|
+
existing_combinations.each do |existing_combination|
|
222
|
+
conditions.each do |condition|
|
223
|
+
new_combinations << (existing_combination + [condition])
|
224
|
+
end
|
225
|
+
end
|
226
|
+
|
227
|
+
new_combinations
|
228
|
+
end
|
229
|
+
|
230
|
+
def create_intersection_fields(combinations, group_config)
|
231
|
+
combinations.map do |condition_combination|
|
232
|
+
# Format name with first condition lowercase, others capitalized
|
233
|
+
field_name = format_intersection_name(condition_combination)
|
234
|
+
create_metric_field(field_name, group_config)
|
235
|
+
end
|
236
|
+
end
|
237
|
+
|
238
|
+
def format_intersection_name(condition_combination)
|
239
|
+
name = condition_combination.first
|
240
|
+
condition_combination[1..].each do |condition|
|
241
|
+
name += condition.capitalize
|
242
|
+
end
|
243
|
+
name
|
244
|
+
end
|
245
|
+
|
246
|
+
def breakout_metrics_fields(group_config)
|
247
|
+
[
|
248
|
+
*countif_fields(group_config),
|
249
|
+
*sumif_fields(group_config)
|
250
|
+
]
|
251
|
+
end
|
252
|
+
|
253
|
+
def countif_fields(group_config)
|
254
|
+
return [] unless group_config.dig("aggregations", "countif")
|
255
|
+
|
256
|
+
[{
|
257
|
+
"name" => group_config["aggregations"]["countif"],
|
258
|
+
"type" => "INTEGER",
|
259
|
+
"mode" => "NULLABLE"
|
260
|
+
}]
|
261
|
+
end
|
262
|
+
|
263
|
+
def sumif_fields(group_config)
|
264
|
+
return [] unless group_config.dig("aggregations", "sumif")
|
265
|
+
|
266
|
+
group_config["aggregations"]["sumif"].keys.map do |metric_name|
|
267
|
+
{
|
268
|
+
"name" => metric_name,
|
269
|
+
"type" => "INTEGER",
|
270
|
+
"mode" => "NULLABLE"
|
271
|
+
}
|
272
|
+
end
|
273
|
+
end
|
274
|
+
end
|
275
|
+
# rubocop:enable Metrics/ClassLength
|
276
|
+
end
|
277
|
+
end
|
@@ -25,85 +25,6 @@ module Manifold
|
|
25
25
|
end
|
26
26
|
end
|
27
27
|
|
28
|
-
# Handles SQL generation for manifold workspaces
|
29
|
-
class SqlGenerator
|
30
|
-
def initialize(name, manifold_yaml)
|
31
|
-
@name = name
|
32
|
-
@manifold_yaml = manifold_yaml
|
33
|
-
end
|
34
|
-
|
35
|
-
def generate_dimensions_merge_sql(source_sql)
|
36
|
-
return unless valid_dimensions_config?
|
37
|
-
|
38
|
-
sql_builder = Terraform::SQLBuilder.new(@name, @manifold_yaml)
|
39
|
-
sql_builder.build_dimensions_merge_sql(source_sql)
|
40
|
-
end
|
41
|
-
|
42
|
-
private
|
43
|
-
|
44
|
-
def valid_dimensions_config?
|
45
|
-
return false unless @manifold_yaml
|
46
|
-
|
47
|
-
!@manifold_yaml["dimensions"]&.dig("merge", "source").nil?
|
48
|
-
end
|
49
|
-
end
|
50
|
-
|
51
|
-
# Handles schema file generation for manifold workspaces
|
52
|
-
class SchemaWriter
|
53
|
-
def initialize(name, vectors, vector_service, manifold_yaml, logger)
|
54
|
-
@name = name
|
55
|
-
@vectors = vectors
|
56
|
-
@vector_service = vector_service
|
57
|
-
@manifold_yaml = manifold_yaml
|
58
|
-
@logger = logger
|
59
|
-
end
|
60
|
-
|
61
|
-
def write_schemas(tables_directory)
|
62
|
-
tables_directory.mkpath
|
63
|
-
write_dimensions_schema(tables_directory)
|
64
|
-
write_manifold_schema(tables_directory)
|
65
|
-
end
|
66
|
-
|
67
|
-
private
|
68
|
-
|
69
|
-
def write_dimensions_schema(tables_directory)
|
70
|
-
dimensions_path = tables_directory.join("dimensions.json")
|
71
|
-
dimensions_path.write(dimensions_schema_json.concat("\n"))
|
72
|
-
end
|
73
|
-
|
74
|
-
def write_manifold_schema(tables_directory)
|
75
|
-
manifold_path = tables_directory.join("manifold.json")
|
76
|
-
manifold_path.write(manifold_schema_json.concat("\n"))
|
77
|
-
end
|
78
|
-
|
79
|
-
def schema_generator
|
80
|
-
@schema_generator ||= SchemaGenerator.new(dimensions_fields, @manifold_yaml)
|
81
|
-
end
|
82
|
-
|
83
|
-
def manifold_schema
|
84
|
-
schema_generator.manifold_schema
|
85
|
-
end
|
86
|
-
|
87
|
-
def dimensions_schema
|
88
|
-
schema_generator.dimensions_schema
|
89
|
-
end
|
90
|
-
|
91
|
-
def dimensions_fields
|
92
|
-
@dimensions_fields ||= @vectors.filter_map do |vector|
|
93
|
-
@logger.info("Loading vector schema for '#{vector}'.")
|
94
|
-
@vector_service.load_vector_schema(vector)
|
95
|
-
end
|
96
|
-
end
|
97
|
-
|
98
|
-
def dimensions_schema_json
|
99
|
-
JSON.pretty_generate(dimensions_schema)
|
100
|
-
end
|
101
|
-
|
102
|
-
def manifold_schema_json
|
103
|
-
JSON.pretty_generate(manifold_schema)
|
104
|
-
end
|
105
|
-
end
|
106
|
-
|
107
28
|
# Encapsulates a single manifold.
|
108
29
|
class Workspace
|
109
30
|
attr_reader :name, :template_path, :logger
|
@@ -131,7 +52,7 @@ module Manifold
|
|
131
52
|
def generate(with_terraform: false)
|
132
53
|
return nil unless manifold_exists? && any_vectors?
|
133
54
|
|
134
|
-
|
55
|
+
write_schemas
|
135
56
|
logger.info("Generated BigQuery dimensions table schema for workspace '#{name}'.")
|
136
57
|
|
137
58
|
return unless with_terraform
|
@@ -177,21 +98,20 @@ module Manifold
|
|
177
98
|
end
|
178
99
|
|
179
100
|
def write_dimensions_merge_sql
|
180
|
-
return unless
|
101
|
+
return unless valid_dimensions_config?
|
181
102
|
|
182
|
-
|
103
|
+
source_sql = File.read(Pathname.pwd.join(manifold_yaml["dimensions"]["merge"]["source"]))
|
104
|
+
sql_builder = Terraform::SQLBuilder.new(name, manifold_yaml)
|
105
|
+
sql = sql_builder.build_dimensions_merge_sql(source_sql)
|
183
106
|
return unless sql
|
184
107
|
|
185
108
|
write_dimensions_merge_sql_file(sql)
|
186
109
|
end
|
187
110
|
|
188
|
-
def
|
189
|
-
manifold_yaml
|
190
|
-
end
|
111
|
+
def valid_dimensions_config?
|
112
|
+
return false unless manifold_yaml
|
191
113
|
|
192
|
-
|
193
|
-
source_sql = File.read(Pathname.pwd.join(manifold_yaml["dimensions"]["merge"]["source"]))
|
194
|
-
SqlGenerator.new(name, manifold_yaml).generate_dimensions_merge_sql(source_sql)
|
114
|
+
!manifold_yaml["dimensions"]&.dig("merge", "source").nil?
|
195
115
|
end
|
196
116
|
|
197
117
|
def write_dimensions_merge_sql_file(sql)
|
@@ -213,9 +133,9 @@ module Manifold
|
|
213
133
|
@manifold_yaml ||= YAML.safe_load_file(manifold_path)
|
214
134
|
end
|
215
135
|
|
216
|
-
def
|
217
|
-
|
218
|
-
|
136
|
+
def write_schemas
|
137
|
+
SchemaManager.new(name, vectors, @vector_service, manifold_yaml, logger)
|
138
|
+
.write_schemas(tables_directory)
|
219
139
|
end
|
220
140
|
|
221
141
|
def any_vectors?
|
@@ -14,9 +14,22 @@ timestamp:
|
|
14
14
|
|
15
15
|
metrics:
|
16
16
|
renders:
|
17
|
+
conditions:
|
18
|
+
mobile: IS_DESKTOP(context.device)
|
19
|
+
desktop: IS_MOBILE(context.device)
|
20
|
+
us: context.geo.country = 'US'
|
21
|
+
global: context.geo.country != 'US'
|
22
|
+
|
17
23
|
breakouts:
|
18
|
-
|
19
|
-
|
24
|
+
device:
|
25
|
+
- mobile
|
26
|
+
- desktop
|
27
|
+
acquisition:
|
28
|
+
- organic
|
29
|
+
- paid
|
30
|
+
region:
|
31
|
+
- us
|
32
|
+
- global
|
20
33
|
|
21
34
|
aggregations:
|
22
35
|
countif: renderCount
|
@@ -24,5 +37,4 @@ metrics:
|
|
24
37
|
sequenceSum:
|
25
38
|
field: context.sequence
|
26
39
|
|
27
|
-
source: my_project.render_metrics
|
28
40
|
filter: timestamp >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 90 DAY)
|
@@ -6,7 +6,7 @@ module Manifold
|
|
6
6
|
class ProjectConfiguration < Configuration
|
7
7
|
attr_reader :workspaces, :provider_version, :skip_provider_config
|
8
8
|
|
9
|
-
DEFAULT_TERRAFORM_GOOGLE_PROVIDER_VERSION = "6.
|
9
|
+
DEFAULT_TERRAFORM_GOOGLE_PROVIDER_VERSION = "6.24.0"
|
10
10
|
|
11
11
|
def initialize(workspaces, provider_version: DEFAULT_TERRAFORM_GOOGLE_PROVIDER_VERSION,
|
12
12
|
skip_provider_config: false)
|
@@ -2,50 +2,11 @@
|
|
2
2
|
|
3
3
|
module Manifold
|
4
4
|
module Terraform
|
5
|
-
# Handles building metrics SQL for manifold routines
|
6
|
-
class MetricsSQLBuilder
|
7
|
-
def initialize(name, manifold_config)
|
8
|
-
@name = name
|
9
|
-
@manifold_config = manifold_config
|
10
|
-
end
|
11
|
-
|
12
|
-
def build_metrics_select
|
13
|
-
<<~SQL
|
14
|
-
SELECT
|
15
|
-
id,
|
16
|
-
timestamp,
|
17
|
-
#{build_metrics_struct}
|
18
|
-
FROM #{build_metric_joins}
|
19
|
-
SQL
|
20
|
-
end
|
21
|
-
|
22
|
-
private
|
23
|
-
|
24
|
-
def build_metrics_struct
|
25
|
-
metric_groups = @manifold_config["metrics"].keys
|
26
|
-
metric_groups.map { |group| "#{group}.metrics #{group}" }.join(",\n ")
|
27
|
-
end
|
28
|
-
|
29
|
-
def build_metric_joins
|
30
|
-
metric_groups = @manifold_config["metrics"]
|
31
|
-
joins = metric_groups.map { |group, config| "#{config["source"]} AS #{group}" }
|
32
|
-
first = joins.shift
|
33
|
-
return first if joins.empty?
|
34
|
-
|
35
|
-
"#{first}\n #{joins.map { |table| "FULL OUTER JOIN #{table} USING (id, timestamp)" }.join("\n ")}"
|
36
|
-
end
|
37
|
-
|
38
|
-
def timestamp_field
|
39
|
-
@manifold_config&.dig("timestamp", "field")
|
40
|
-
end
|
41
|
-
end
|
42
|
-
|
43
5
|
# Handles building SQL for manifold routines
|
44
6
|
class SQLBuilder
|
45
7
|
def initialize(name, manifold_config)
|
46
8
|
@name = name
|
47
9
|
@manifold_config = manifold_config
|
48
|
-
@metrics_builder = MetricsSQLBuilder.new(name, manifold_config)
|
49
10
|
end
|
50
11
|
|
51
12
|
def build_manifold_merge_sql
|
@@ -75,22 +36,21 @@ module Manifold
|
|
75
36
|
private
|
76
37
|
|
77
38
|
def valid_config?
|
78
|
-
|
79
|
-
end
|
80
|
-
|
81
|
-
def source_table
|
82
|
-
first_group = @manifold_config["metrics"]&.values&.first
|
83
|
-
first_group&.dig("source")
|
39
|
+
timestamp_field && @manifold_config["metrics"] && !@manifold_config["metrics"].empty?
|
84
40
|
end
|
85
41
|
|
86
42
|
def timestamp_field
|
87
43
|
@manifold_config&.dig("timestamp", "field")
|
88
44
|
end
|
89
45
|
|
46
|
+
def metrics_table_name(group_name)
|
47
|
+
"#{group_name.capitalize}Metrics"
|
48
|
+
end
|
49
|
+
|
90
50
|
def build_source_query
|
91
51
|
<<~SQL
|
92
52
|
WITH Metrics AS (
|
93
|
-
#{
|
53
|
+
#{build_metrics_select}
|
94
54
|
)
|
95
55
|
|
96
56
|
SELECT
|
@@ -117,23 +77,65 @@ module Manifold
|
|
117
77
|
INSERT ROW;
|
118
78
|
SQL
|
119
79
|
end
|
80
|
+
|
81
|
+
# Metrics SQL building methods
|
82
|
+
def build_metrics_select
|
83
|
+
<<~SQL
|
84
|
+
SELECT
|
85
|
+
id,
|
86
|
+
timestamp,
|
87
|
+
#{build_metrics_struct}
|
88
|
+
FROM #{build_metric_joins}
|
89
|
+
SQL
|
90
|
+
end
|
91
|
+
|
92
|
+
def build_metrics_struct
|
93
|
+
metric_groups = @manifold_config["metrics"].keys
|
94
|
+
metric_groups.map { |group| "#{group}.metrics #{group}" }.join(",\n ")
|
95
|
+
end
|
96
|
+
|
97
|
+
def build_metric_joins
|
98
|
+
metric_groups = @manifold_config["metrics"]
|
99
|
+
joins = metric_groups.map do |group, config|
|
100
|
+
table = "#{@name}.#{metrics_table_name(group)}"
|
101
|
+
filter = config["filter"] ? " WHERE #{config["filter"]}" : ""
|
102
|
+
"(SELECT * FROM #{table}#{filter}) AS #{group}"
|
103
|
+
end
|
104
|
+
first = joins.shift
|
105
|
+
return first if joins.empty?
|
106
|
+
|
107
|
+
"#{first}\n #{joins.map { |table| "FULL OUTER JOIN #{table} USING (id, timestamp)" }.join("\n ")}"
|
108
|
+
end
|
120
109
|
end
|
121
110
|
|
122
111
|
# Handles building table configurations
|
123
112
|
class TableConfigBuilder
|
124
|
-
def initialize(name)
|
113
|
+
def initialize(name, manifold_config = nil)
|
125
114
|
@name = name
|
115
|
+
@manifold_config = manifold_config
|
126
116
|
end
|
127
117
|
|
128
118
|
def build_table_configs
|
129
|
-
{
|
119
|
+
configs = {
|
130
120
|
"dimensions" => dimensions_table_config,
|
131
121
|
"manifold" => manifold_table_config
|
132
122
|
}
|
123
|
+
|
124
|
+
if @manifold_config&.dig("metrics")
|
125
|
+
@manifold_config["metrics"].each_key do |group_name|
|
126
|
+
configs[metrics_table_name(group_name).downcase] = metrics_table_config(group_name)
|
127
|
+
end
|
128
|
+
end
|
129
|
+
|
130
|
+
configs
|
133
131
|
end
|
134
132
|
|
135
133
|
private
|
136
134
|
|
135
|
+
def metrics_table_name(group_name)
|
136
|
+
"#{group_name.capitalize}Metrics"
|
137
|
+
end
|
138
|
+
|
137
139
|
def dimensions_table_config
|
138
140
|
build_table_config("Dimensions")
|
139
141
|
end
|
@@ -142,12 +144,18 @@ module Manifold
|
|
142
144
|
build_table_config("Manifold")
|
143
145
|
end
|
144
146
|
|
145
|
-
def
|
147
|
+
def metrics_table_config(group_name)
|
148
|
+
titlecased_name = metrics_table_name(group_name)
|
149
|
+
build_table_config(titlecased_name, "metrics/#{group_name}.json")
|
150
|
+
end
|
151
|
+
|
152
|
+
def build_table_config(table_id, schema_path = nil)
|
153
|
+
schema_path ||= "#{table_id.downcase}.json"
|
146
154
|
{
|
147
155
|
"dataset_id" => @name,
|
148
156
|
"project" => "${var.project_id}",
|
149
157
|
"table_id" => table_id,
|
150
|
-
"schema" => "${file(\"${path.module}/tables/#{
|
158
|
+
"schema" => "${file(\"${path.module}/tables/#{schema_path}\")}",
|
151
159
|
"depends_on" => ["google_bigquery_dataset.#{@name}"]
|
152
160
|
}
|
153
161
|
end
|
@@ -174,7 +182,7 @@ module Manifold
|
|
174
182
|
"variable" => variables_block,
|
175
183
|
"resource" => {
|
176
184
|
"google_bigquery_dataset" => dataset_config,
|
177
|
-
"google_bigquery_table" => TableConfigBuilder.new(name).build_table_configs,
|
185
|
+
"google_bigquery_table" => TableConfigBuilder.new(name, @manifold_config).build_table_configs,
|
178
186
|
"google_bigquery_routine" => routine_config
|
179
187
|
}.compact
|
180
188
|
}
|
data/lib/manifold/version.rb
CHANGED
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: manifold-cli
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0
|
4
|
+
version: 0.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- claytongentry
|
8
8
|
bindir: bin
|
9
9
|
cert_chain: []
|
10
|
-
date: 2025-
|
10
|
+
date: 2025-03-31 00:00:00.000000000 Z
|
11
11
|
dependencies:
|
12
12
|
- !ruby/object:Gem::Dependency
|
13
13
|
name: thor
|
@@ -43,7 +43,7 @@ files:
|
|
43
43
|
- lib/manifold.rb
|
44
44
|
- lib/manifold/api.rb
|
45
45
|
- lib/manifold/api/project.rb
|
46
|
-
- lib/manifold/api/
|
46
|
+
- lib/manifold/api/schema_manager.rb
|
47
47
|
- lib/manifold/api/vector.rb
|
48
48
|
- lib/manifold/api/workspace.rb
|
49
49
|
- lib/manifold/cli.rb
|
@@ -1,89 +0,0 @@
|
|
1
|
-
# frozen_string_literal: true
|
2
|
-
|
3
|
-
module Manifold
|
4
|
-
module API
|
5
|
-
# Handles schema generation for Manifold tables
|
6
|
-
class SchemaGenerator
|
7
|
-
def initialize(dimensions_fields, manifold_yaml)
|
8
|
-
@dimensions_fields = dimensions_fields
|
9
|
-
@manifold_yaml = manifold_yaml
|
10
|
-
end
|
11
|
-
|
12
|
-
def dimensions_schema
|
13
|
-
[
|
14
|
-
{ "type" => "STRING", "name" => "id", "mode" => "REQUIRED" },
|
15
|
-
{ "type" => "RECORD", "name" => "dimensions", "mode" => "REQUIRED",
|
16
|
-
"fields" => @dimensions_fields }
|
17
|
-
]
|
18
|
-
end
|
19
|
-
|
20
|
-
def manifold_schema
|
21
|
-
[
|
22
|
-
{ "type" => "STRING", "name" => "id", "mode" => "REQUIRED" },
|
23
|
-
{ "type" => "TIMESTAMP", "name" => "timestamp", "mode" => "REQUIRED" },
|
24
|
-
{ "type" => "RECORD", "name" => "dimensions", "mode" => "REQUIRED",
|
25
|
-
"fields" => @dimensions_fields },
|
26
|
-
{ "type" => "RECORD", "name" => "metrics", "mode" => "REQUIRED",
|
27
|
-
"fields" => metrics_fields }
|
28
|
-
]
|
29
|
-
end
|
30
|
-
|
31
|
-
private
|
32
|
-
|
33
|
-
def metrics_fields
|
34
|
-
return [] unless @manifold_yaml["metrics"]
|
35
|
-
|
36
|
-
@manifold_yaml["metrics"].map do |group_name, group_config|
|
37
|
-
{
|
38
|
-
"name" => group_name,
|
39
|
-
"type" => "RECORD",
|
40
|
-
"mode" => "NULLABLE",
|
41
|
-
"fields" => group_metrics_fields(group_config)
|
42
|
-
}
|
43
|
-
end
|
44
|
-
end
|
45
|
-
|
46
|
-
def group_metrics_fields(group_config)
|
47
|
-
return [] unless group_config["breakouts"] && group_config["aggregations"]
|
48
|
-
|
49
|
-
group_config["breakouts"].map do |breakout_name, _breakout_config|
|
50
|
-
{
|
51
|
-
"name" => breakout_name,
|
52
|
-
"type" => "RECORD",
|
53
|
-
"mode" => "NULLABLE",
|
54
|
-
"fields" => breakout_metrics_fields(group_config)
|
55
|
-
}
|
56
|
-
end
|
57
|
-
end
|
58
|
-
|
59
|
-
def breakout_metrics_fields(group_config)
|
60
|
-
[
|
61
|
-
*countif_fields(group_config),
|
62
|
-
*sumif_fields(group_config)
|
63
|
-
]
|
64
|
-
end
|
65
|
-
|
66
|
-
def countif_fields(group_config)
|
67
|
-
return [] unless group_config.dig("aggregations", "countif")
|
68
|
-
|
69
|
-
[{
|
70
|
-
"name" => group_config["aggregations"]["countif"],
|
71
|
-
"type" => "INTEGER",
|
72
|
-
"mode" => "NULLABLE"
|
73
|
-
}]
|
74
|
-
end
|
75
|
-
|
76
|
-
def sumif_fields(group_config)
|
77
|
-
return [] unless group_config.dig("aggregations", "sumif")
|
78
|
-
|
79
|
-
group_config["aggregations"]["sumif"].keys.map do |metric_name|
|
80
|
-
{
|
81
|
-
"name" => metric_name,
|
82
|
-
"type" => "INTEGER",
|
83
|
-
"mode" => "NULLABLE"
|
84
|
-
}
|
85
|
-
end
|
86
|
-
end
|
87
|
-
end
|
88
|
-
end
|
89
|
-
end
|