manifold-cli 0.0.16 → 0.0.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/manifold/api/schema_generator.rb +26 -21
- data/lib/manifold/api/workspace.rb +116 -36
- data/lib/manifold/cli.rb +1 -1
- data/lib/manifold/templates/workspace_template.yml +26 -15
- data/lib/manifold/terraform/workspace_configuration.rb +82 -77
- data/lib/manifold/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 61582d1f6df6c8a674f9ef9b2374ae231fc98da47435ea5893d5077cb4dad34e
|
4
|
+
data.tar.gz: ab91d3938b7a382ac003a467cd04d19cc6059804fe21e099e7e2aa60675eed3a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 5cc06d2a611b1b29edbb466ab20cf8b82d4b99d609f57b71bd5e7f6098e7fcbd636c46a662306b8318c429148fbba3b4b6d6ddc3c603cddcb53ff81c9fe50f4c
|
7
|
+
data.tar.gz: b8e44688d8e47628dc718bf992eb28114fde80a7cd60975e7a49cf91ce05fdf3ed2b6bc349c73332801ed5934b0a56a6f71ca100ad4b02df763c2f67cd2a3f8b
|
@@ -4,8 +4,6 @@ module Manifold
|
|
4
4
|
module API
|
5
5
|
# Handles schema generation for Manifold tables
|
6
6
|
class SchemaGenerator
|
7
|
-
VALID_OPERATORS = %w[AND OR NOT NAND NOR XOR XNOR].freeze
|
8
|
-
|
9
7
|
def initialize(dimensions_fields, manifold_yaml)
|
10
8
|
@dimensions_fields = dimensions_fields
|
11
9
|
@manifold_yaml = manifold_yaml
|
@@ -33,39 +31,52 @@ module Manifold
|
|
33
31
|
private
|
34
32
|
|
35
33
|
def metrics_fields
|
36
|
-
return [] unless @manifold_yaml["
|
34
|
+
return [] unless @manifold_yaml["metrics"]
|
35
|
+
|
36
|
+
@manifold_yaml["metrics"].map do |group_name, group_config|
|
37
|
+
{
|
38
|
+
"name" => group_name,
|
39
|
+
"type" => "RECORD",
|
40
|
+
"mode" => "NULLABLE",
|
41
|
+
"fields" => group_metrics_fields(group_config)
|
42
|
+
}
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def group_metrics_fields(group_config)
|
47
|
+
return [] unless group_config["breakouts"] && group_config["aggregations"]
|
37
48
|
|
38
|
-
|
49
|
+
group_config["breakouts"].map do |breakout_name, _breakout_config|
|
39
50
|
{
|
40
|
-
"name" =>
|
51
|
+
"name" => breakout_name,
|
41
52
|
"type" => "RECORD",
|
42
53
|
"mode" => "NULLABLE",
|
43
|
-
"fields" =>
|
54
|
+
"fields" => breakout_metrics_fields(group_config)
|
44
55
|
}
|
45
56
|
end
|
46
57
|
end
|
47
58
|
|
48
|
-
def
|
59
|
+
def breakout_metrics_fields(group_config)
|
49
60
|
[
|
50
|
-
*countif_fields,
|
51
|
-
*sumif_fields
|
61
|
+
*countif_fields(group_config),
|
62
|
+
*sumif_fields(group_config)
|
52
63
|
]
|
53
64
|
end
|
54
65
|
|
55
|
-
def countif_fields
|
56
|
-
return [] unless
|
66
|
+
def countif_fields(group_config)
|
67
|
+
return [] unless group_config.dig("aggregations", "countif")
|
57
68
|
|
58
69
|
[{
|
59
|
-
"name" =>
|
70
|
+
"name" => group_config["aggregations"]["countif"],
|
60
71
|
"type" => "INTEGER",
|
61
72
|
"mode" => "NULLABLE"
|
62
73
|
}]
|
63
74
|
end
|
64
75
|
|
65
|
-
def sumif_fields
|
66
|
-
return [] unless
|
76
|
+
def sumif_fields(group_config)
|
77
|
+
return [] unless group_config.dig("aggregations", "sumif")
|
67
78
|
|
68
|
-
|
79
|
+
group_config["aggregations"]["sumif"].keys.map do |metric_name|
|
69
80
|
{
|
70
81
|
"name" => metric_name,
|
71
82
|
"type" => "INTEGER",
|
@@ -73,12 +84,6 @@ module Manifold
|
|
73
84
|
}
|
74
85
|
end
|
75
86
|
end
|
76
|
-
|
77
|
-
def validate_operator!(operator)
|
78
|
-
return if VALID_OPERATORS.include?(operator)
|
79
|
-
|
80
|
-
raise ArgumentError, "Invalid operator: #{operator}. Valid operators are: #{VALID_OPERATORS.join(", ")}"
|
81
|
-
end
|
82
87
|
end
|
83
88
|
end
|
84
89
|
end
|
@@ -19,12 +19,91 @@ module Manifold
|
|
19
19
|
vector_config = @vector_service.load_vector_config(vector)
|
20
20
|
config.add_vector(vector_config)
|
21
21
|
end
|
22
|
-
config.
|
22
|
+
config.dimensions_config = @manifold_yaml["dimensions"]&.fetch("merge", nil) if @manifold_yaml["dimensions"]
|
23
23
|
config.manifold_config = @manifold_yaml
|
24
24
|
config.write(path)
|
25
25
|
end
|
26
26
|
end
|
27
27
|
|
28
|
+
# Handles SQL generation for manifold workspaces
|
29
|
+
class SqlGenerator
|
30
|
+
def initialize(name, manifold_yaml)
|
31
|
+
@name = name
|
32
|
+
@manifold_yaml = manifold_yaml
|
33
|
+
end
|
34
|
+
|
35
|
+
def generate_dimensions_merge_sql(source_sql)
|
36
|
+
return unless valid_dimensions_config?
|
37
|
+
|
38
|
+
sql_builder = Terraform::SQLBuilder.new(@name, @manifold_yaml)
|
39
|
+
sql_builder.build_dimensions_merge_sql(source_sql)
|
40
|
+
end
|
41
|
+
|
42
|
+
private
|
43
|
+
|
44
|
+
def valid_dimensions_config?
|
45
|
+
return false unless @manifold_yaml
|
46
|
+
|
47
|
+
!@manifold_yaml["dimensions"]&.dig("merge", "source").nil?
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
# Handles schema file generation for manifold workspaces
|
52
|
+
class SchemaWriter
|
53
|
+
def initialize(name, vectors, vector_service, manifold_yaml, logger)
|
54
|
+
@name = name
|
55
|
+
@vectors = vectors
|
56
|
+
@vector_service = vector_service
|
57
|
+
@manifold_yaml = manifold_yaml
|
58
|
+
@logger = logger
|
59
|
+
end
|
60
|
+
|
61
|
+
def write_schemas(tables_directory)
|
62
|
+
tables_directory.mkpath
|
63
|
+
write_dimensions_schema(tables_directory)
|
64
|
+
write_manifold_schema(tables_directory)
|
65
|
+
end
|
66
|
+
|
67
|
+
private
|
68
|
+
|
69
|
+
def write_dimensions_schema(tables_directory)
|
70
|
+
dimensions_path = tables_directory.join("dimensions.json")
|
71
|
+
dimensions_path.write(dimensions_schema_json.concat("\n"))
|
72
|
+
end
|
73
|
+
|
74
|
+
def write_manifold_schema(tables_directory)
|
75
|
+
manifold_path = tables_directory.join("manifold.json")
|
76
|
+
manifold_path.write(manifold_schema_json.concat("\n"))
|
77
|
+
end
|
78
|
+
|
79
|
+
def schema_generator
|
80
|
+
@schema_generator ||= SchemaGenerator.new(dimensions_fields, @manifold_yaml)
|
81
|
+
end
|
82
|
+
|
83
|
+
def manifold_schema
|
84
|
+
schema_generator.manifold_schema
|
85
|
+
end
|
86
|
+
|
87
|
+
def dimensions_schema
|
88
|
+
schema_generator.dimensions_schema
|
89
|
+
end
|
90
|
+
|
91
|
+
def dimensions_fields
|
92
|
+
@dimensions_fields ||= @vectors.filter_map do |vector|
|
93
|
+
@logger.info("Loading vector schema for '#{vector}'.")
|
94
|
+
@vector_service.load_vector_schema(vector)
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
def dimensions_schema_json
|
99
|
+
JSON.pretty_generate(dimensions_schema)
|
100
|
+
end
|
101
|
+
|
102
|
+
def manifold_schema_json
|
103
|
+
JSON.pretty_generate(manifold_schema)
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
28
107
|
# Encapsulates a single manifold.
|
29
108
|
class Workspace
|
30
109
|
attr_reader :name, :template_path, :logger
|
@@ -52,13 +131,13 @@ module Manifold
|
|
52
131
|
def generate(with_terraform: false)
|
53
132
|
return nil unless manifold_exists? && any_vectors?
|
54
133
|
|
55
|
-
|
56
|
-
generate_dimensions
|
57
|
-
generate_manifold
|
134
|
+
generate_schemas
|
58
135
|
logger.info("Generated BigQuery dimensions table schema for workspace '#{name}'.")
|
59
136
|
|
60
137
|
return unless with_terraform
|
61
138
|
|
139
|
+
write_manifold_merge_sql
|
140
|
+
write_dimensions_merge_sql
|
62
141
|
generate_terraform
|
63
142
|
logger.info("Generated Terraform configuration for workspace '#{name}'.")
|
64
143
|
end
|
@@ -89,53 +168,58 @@ module Manifold
|
|
89
168
|
directory.join("main.tf.json")
|
90
169
|
end
|
91
170
|
|
92
|
-
|
171
|
+
def write_manifold_merge_sql
|
172
|
+
return unless manifold_file
|
93
173
|
|
94
|
-
|
95
|
-
|
96
|
-
|
174
|
+
sql_builder = Terraform::SQLBuilder.new(name, manifold_yaml)
|
175
|
+
metrics_builder = Terraform::MetricsBuilder.new(manifold_yaml)
|
176
|
+
sql = sql_builder.build_manifold_merge_sql(metrics_builder) do
|
177
|
+
metrics_builder.build_metrics_struct
|
178
|
+
end
|
97
179
|
|
98
|
-
|
99
|
-
@manifold_yaml ||= YAML.safe_load_file(manifold_path)
|
180
|
+
routines_directory.join("merge_manifold.sql").write(sql)
|
100
181
|
end
|
101
182
|
|
102
|
-
def
|
103
|
-
|
104
|
-
end
|
183
|
+
def write_dimensions_merge_sql
|
184
|
+
return unless dimensions_merge_source_exists?
|
105
185
|
|
106
|
-
|
107
|
-
|
186
|
+
sql = generate_dimensions_merge_sql
|
187
|
+
return unless sql
|
188
|
+
|
189
|
+
write_dimensions_merge_sql_file(sql)
|
108
190
|
end
|
109
191
|
|
110
|
-
def
|
111
|
-
|
192
|
+
def dimensions_merge_source_exists?
|
193
|
+
manifold_yaml["dimensions"]&.dig("merge", "source")
|
112
194
|
end
|
113
195
|
|
114
|
-
def
|
115
|
-
|
196
|
+
def generate_dimensions_merge_sql
|
197
|
+
source_sql = File.read(Pathname.pwd.join(manifold_yaml["dimensions"]["merge"]["source"]))
|
198
|
+
SqlGenerator.new(name, manifold_yaml).generate_dimensions_merge_sql(source_sql)
|
116
199
|
end
|
117
200
|
|
118
|
-
def
|
119
|
-
|
201
|
+
def write_dimensions_merge_sql_file(sql)
|
202
|
+
routines_directory.mkpath
|
203
|
+
dimensions_merge_sql_path.write(sql)
|
120
204
|
end
|
121
205
|
|
122
|
-
def
|
123
|
-
|
206
|
+
def dimensions_merge_sql_path
|
207
|
+
routines_directory.join("merge_dimensions.sql")
|
124
208
|
end
|
125
209
|
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
end
|
210
|
+
private
|
211
|
+
|
212
|
+
def directory
|
213
|
+
Pathname.pwd.join("workspaces", name)
|
131
214
|
end
|
132
215
|
|
133
|
-
def
|
134
|
-
|
216
|
+
def manifold_yaml
|
217
|
+
@manifold_yaml ||= YAML.safe_load_file(manifold_path)
|
135
218
|
end
|
136
219
|
|
137
|
-
def
|
138
|
-
|
220
|
+
def generate_schemas
|
221
|
+
SchemaWriter.new(name, vectors, @vector_service, manifold_yaml, logger)
|
222
|
+
.write_schemas(tables_directory)
|
139
223
|
end
|
140
224
|
|
141
225
|
def any_vectors?
|
@@ -151,10 +235,6 @@ module Manifold
|
|
151
235
|
terraform_generator.manifold_config = manifold_yaml
|
152
236
|
terraform_generator.generate(terraform_main_path)
|
153
237
|
end
|
154
|
-
|
155
|
-
def manifold_schema_json
|
156
|
-
JSON.pretty_generate(manifold_schema)
|
157
|
-
end
|
158
238
|
end
|
159
239
|
end
|
160
240
|
end
|
data/lib/manifold/cli.rb
CHANGED
@@ -4,7 +4,7 @@ module Manifold
|
|
4
4
|
# CLI provides command line interface functionality
|
5
5
|
# for creating and managing umbrella projects for data management.
|
6
6
|
class CLI < Thor
|
7
|
-
attr_accessor :logger
|
7
|
+
attr_accessor :logger
|
8
8
|
|
9
9
|
def initialize(*args, logger: Logger.new($stdout))
|
10
10
|
super(*args)
|
@@ -12,20 +12,31 @@ timestamp:
|
|
12
12
|
interval: HOUR
|
13
13
|
field: timestamp
|
14
14
|
|
15
|
-
contexts:
|
16
|
-
paid: IS_PAID(context.location)
|
17
|
-
organic: IS_ORGANIC(context.location)
|
18
|
-
paidOrganic:
|
19
|
-
fields:
|
20
|
-
- paid
|
21
|
-
- organic
|
22
|
-
operator: AND
|
23
|
-
|
24
15
|
metrics:
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
16
|
+
renders:
|
17
|
+
breakouts:
|
18
|
+
paid: IS_PAID(context.location)
|
19
|
+
organic: IS_ORGANIC(context.location)
|
20
|
+
|
21
|
+
aggregations:
|
22
|
+
countif: renderCount
|
23
|
+
sumif:
|
24
|
+
sequenceSum:
|
25
|
+
field: context.sequence
|
26
|
+
|
27
|
+
source: my_project.my_dataset.my_table
|
28
|
+
filter: timestamp >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 90 DAY)
|
29
|
+
|
30
|
+
taps:
|
31
|
+
breakouts:
|
32
|
+
paid: IS_PAID(context.location)
|
33
|
+
organic: IS_ORGANIC(context.location)
|
34
|
+
|
35
|
+
aggregations:
|
36
|
+
countif: tapCount
|
37
|
+
sumif:
|
38
|
+
sequenceSum:
|
39
|
+
field: context.sequence
|
29
40
|
|
30
|
-
source: my_project.my_dataset.my_table
|
31
|
-
filter: timestamp >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 90 DAY)
|
41
|
+
source: my_project.my_dataset.my_table
|
42
|
+
filter: timestamp >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 90 DAY)
|
@@ -9,48 +9,76 @@ module Manifold
|
|
9
9
|
end
|
10
10
|
|
11
11
|
def build_metrics_struct
|
12
|
-
return "" unless @manifold_config
|
12
|
+
return "" unless @manifold_config["metrics"]
|
13
13
|
|
14
|
-
|
15
|
-
|
16
|
-
metrics = build_context_metrics(condition)
|
17
|
-
"STRUCT(#{metrics}) AS #{name}"
|
14
|
+
metric_groups = @manifold_config["metrics"].map do |group_name, group_config|
|
15
|
+
build_group_struct(group_name, group_config)
|
18
16
|
end
|
19
17
|
|
20
|
-
|
18
|
+
metric_groups.join(",\n")
|
21
19
|
end
|
22
20
|
|
23
21
|
private
|
24
22
|
|
25
|
-
def
|
23
|
+
def build_group_struct(group_name, group_config)
|
24
|
+
return "" unless valid_group_config?(group_config)
|
25
|
+
|
26
|
+
breakout_structs = build_breakout_structs(group_config)
|
27
|
+
return "" if breakout_structs.empty?
|
28
|
+
|
29
|
+
"\tSTRUCT(\n#{breakout_structs.join(",\n")}\n\t) AS #{group_name}"
|
30
|
+
end
|
31
|
+
|
32
|
+
def valid_group_config?(group_config)
|
33
|
+
group_config["breakouts"] &&
|
34
|
+
group_config["aggregations"] &&
|
35
|
+
!group_config["breakouts"].empty? &&
|
36
|
+
!group_config["aggregations"].empty?
|
37
|
+
end
|
38
|
+
|
39
|
+
def build_breakout_structs(group_config)
|
40
|
+
group_config["breakouts"].map do |name, config|
|
41
|
+
build_breakout_struct(name, config, group_config)
|
42
|
+
end.compact
|
43
|
+
end
|
44
|
+
|
45
|
+
def build_breakout_struct(name, config, group_config)
|
46
|
+
condition = build_breakout_condition(name, config, group_config)
|
47
|
+
metrics = build_breakout_metrics(group_config, condition)
|
48
|
+
return if metrics.empty?
|
49
|
+
|
50
|
+
"\t\tSTRUCT(\n\t\t\t#{metrics}\n\t\t) AS #{name}"
|
51
|
+
end
|
52
|
+
|
53
|
+
def build_breakout_metrics(group_config, condition)
|
26
54
|
metrics = []
|
27
|
-
add_count_metrics(metrics, condition)
|
28
|
-
add_sum_metrics(metrics, condition)
|
29
|
-
metrics.join(",\n")
|
55
|
+
add_count_metrics(metrics, group_config, condition)
|
56
|
+
add_sum_metrics(metrics, group_config, condition)
|
57
|
+
metrics.join(",\n\t\t\t")
|
30
58
|
end
|
31
59
|
|
32
|
-
def add_count_metrics(metrics, condition)
|
33
|
-
return unless
|
60
|
+
def add_count_metrics(metrics, group_config, condition)
|
61
|
+
return unless group_config.dig("aggregations", "countif")
|
34
62
|
|
35
|
-
metrics << "COUNTIF(#{condition}) AS #{
|
63
|
+
metrics << "COUNTIF(#{condition}) AS #{group_config["aggregations"]["countif"]}"
|
36
64
|
end
|
37
65
|
|
38
|
-
def add_sum_metrics(metrics, condition)
|
39
|
-
|
66
|
+
def add_sum_metrics(metrics, group_config, condition)
|
67
|
+
group_config.dig("aggregations", "sumif")&.each do |name, config|
|
40
68
|
metrics << "SUM(IF(#{condition}, #{config["field"]}, 0)) AS #{name}"
|
41
69
|
end
|
42
70
|
end
|
43
71
|
|
44
|
-
def
|
72
|
+
def build_breakout_condition(_name, config, group_config)
|
45
73
|
return config unless config.is_a?(Hash)
|
46
74
|
|
47
75
|
operator = config["operator"]
|
48
76
|
fields = config["fields"]
|
49
|
-
build_operator_condition(operator, fields)
|
77
|
+
build_operator_condition(operator, fields, group_config)
|
50
78
|
end
|
51
79
|
|
52
|
-
def build_operator_condition(operator, fields)
|
53
|
-
conditions = fields.map { |f|
|
80
|
+
def build_operator_condition(operator, fields, group_config)
|
81
|
+
conditions = fields.map { |f| group_config["breakouts"][f] }
|
54
82
|
case operator
|
55
83
|
when "AND", "OR" then join_conditions(conditions, operator)
|
56
84
|
when "NOT" then negate_condition(conditions.first)
|
@@ -117,7 +145,27 @@ module Manifold
|
|
117
145
|
private
|
118
146
|
|
119
147
|
def valid_config?
|
120
|
-
source_table && timestamp_field
|
148
|
+
source_table && timestamp_field && @manifold_config["metrics"]
|
149
|
+
end
|
150
|
+
|
151
|
+
def source_table
|
152
|
+
first_group = @manifold_config["metrics"]&.values&.first
|
153
|
+
first_group&.dig("source")
|
154
|
+
end
|
155
|
+
|
156
|
+
def interval
|
157
|
+
@manifold_config&.dig("timestamp", "interval") || "DAY"
|
158
|
+
end
|
159
|
+
|
160
|
+
def where_clause
|
161
|
+
first_group = @manifold_config["metrics"]&.values&.first
|
162
|
+
return "" unless first_group&.dig("filter")
|
163
|
+
|
164
|
+
"WHERE #{first_group["filter"]}"
|
165
|
+
end
|
166
|
+
|
167
|
+
def timestamp_field
|
168
|
+
@manifold_config&.dig("timestamp", "field")
|
121
169
|
end
|
122
170
|
|
123
171
|
def build_metrics_cte(&)
|
@@ -131,12 +179,12 @@ module Manifold
|
|
131
179
|
def build_metrics_select(&block)
|
132
180
|
<<~SQL
|
133
181
|
SELECT
|
134
|
-
|
182
|
+
id,
|
135
183
|
TIMESTAMP_TRUNC(#{timestamp_field}, #{interval}) timestamp,
|
136
184
|
STRUCT(
|
137
185
|
#{block.call}
|
138
186
|
) AS metrics
|
139
|
-
FROM
|
187
|
+
FROM #{source_table}
|
140
188
|
#{where_clause}
|
141
189
|
GROUP BY 1, 2
|
142
190
|
SQL
|
@@ -144,7 +192,11 @@ module Manifold
|
|
144
192
|
|
145
193
|
def build_final_select
|
146
194
|
<<~SQL
|
147
|
-
SELECT
|
195
|
+
SELECT
|
196
|
+
id,
|
197
|
+
timestamp,
|
198
|
+
Dimensions.dimensions,
|
199
|
+
Metrics.metrics
|
148
200
|
FROM Metrics
|
149
201
|
LEFT JOIN #{@name}.Dimensions USING (id)
|
150
202
|
SQL
|
@@ -160,24 +212,6 @@ module Manifold
|
|
160
212
|
INSERT ROW;
|
161
213
|
SQL
|
162
214
|
end
|
163
|
-
|
164
|
-
def source_table
|
165
|
-
@manifold_config["source"]
|
166
|
-
end
|
167
|
-
|
168
|
-
def interval
|
169
|
-
@manifold_config&.dig("timestamp", "interval") || "DAY"
|
170
|
-
end
|
171
|
-
|
172
|
-
def where_clause
|
173
|
-
return "" unless @manifold_config["filter"]
|
174
|
-
|
175
|
-
"WHERE #{@manifold_config["filter"]}"
|
176
|
-
end
|
177
|
-
|
178
|
-
def timestamp_field
|
179
|
-
@manifold_config&.dig("timestamp", "field")
|
180
|
-
end
|
181
215
|
end
|
182
216
|
|
183
217
|
# Handles building table configurations
|
@@ -217,13 +251,13 @@ module Manifold
|
|
217
251
|
# Represents a Terraform configuration for a Manifold workspace.
|
218
252
|
class WorkspaceConfiguration < Configuration
|
219
253
|
attr_reader :name
|
220
|
-
attr_writer :
|
254
|
+
attr_writer :dimensions_config, :manifold_config
|
221
255
|
|
222
256
|
def initialize(name)
|
223
257
|
super()
|
224
258
|
@name = name
|
225
259
|
@vectors = []
|
226
|
-
@
|
260
|
+
@dimensions_config = nil
|
227
261
|
end
|
228
262
|
|
229
263
|
def add_vector(vector_config)
|
@@ -264,15 +298,15 @@ module Manifold
|
|
264
298
|
|
265
299
|
def routine_config
|
266
300
|
routines = {
|
267
|
-
"merge_dimensions" => dimensions_routine_attributes
|
268
|
-
"merge_manifold" => manifold_routine_attributes
|
301
|
+
"merge_dimensions" => dimensions_routine_attributes
|
302
|
+
# "merge_manifold" => manifold_routine_attributes
|
269
303
|
}.compact
|
270
304
|
|
271
305
|
routines.empty? ? nil : routines
|
272
306
|
end
|
273
307
|
|
274
308
|
def dimensions_routine_attributes
|
275
|
-
return nil if @vectors.empty? || @
|
309
|
+
return nil if @vectors.empty? || @dimensions_config.nil?
|
276
310
|
|
277
311
|
{
|
278
312
|
"dataset_id" => name,
|
@@ -280,51 +314,22 @@ module Manifold
|
|
280
314
|
"routine_id" => "merge_dimensions",
|
281
315
|
"routine_type" => "PROCEDURE",
|
282
316
|
"language" => "SQL",
|
283
|
-
"definition_body" =>
|
317
|
+
"definition_body" => "${file(\"${path.module}/routines/merge_dimensions.sql\")}",
|
284
318
|
"depends_on" => ["google_bigquery_dataset.#{name}"]
|
285
319
|
}
|
286
320
|
end
|
287
321
|
|
288
|
-
def dimensions_merge_routine
|
289
|
-
return "" if @vectors.empty? || @merge_config.nil?
|
290
|
-
|
291
|
-
source_sql = File.read(Pathname.pwd.join(@merge_config["source"]))
|
292
|
-
SQLBuilder.new(name, @manifold_config).build_dimensions_merge_sql(source_sql)
|
293
|
-
end
|
294
|
-
|
295
322
|
def manifold_routine_attributes
|
296
|
-
return nil unless valid_manifold_config?
|
297
|
-
|
298
323
|
{
|
299
324
|
"dataset_id" => name,
|
300
325
|
"project" => "${var.project_id}",
|
301
326
|
"routine_id" => "merge_manifold",
|
302
327
|
"routine_type" => "PROCEDURE",
|
303
328
|
"language" => "SQL",
|
304
|
-
"definition_body" =>
|
329
|
+
"definition_body" => "${file(\"${path.module}/routines/merge_manifold.sql\")}",
|
305
330
|
"depends_on" => ["google_bigquery_dataset.#{name}"]
|
306
331
|
}
|
307
332
|
end
|
308
|
-
|
309
|
-
def manifold_merge_routine
|
310
|
-
metrics_builder = MetricsBuilder.new(@manifold_config)
|
311
|
-
sql_builder = SQLBuilder.new(name, @manifold_config)
|
312
|
-
sql_builder.build_manifold_merge_sql(metrics_builder) do
|
313
|
-
metrics_builder.build_metrics_struct
|
314
|
-
end
|
315
|
-
end
|
316
|
-
|
317
|
-
def valid_manifold_config?
|
318
|
-
return false unless @manifold_config
|
319
|
-
|
320
|
-
required_fields_present?
|
321
|
-
end
|
322
|
-
|
323
|
-
def required_fields_present?
|
324
|
-
%w[source timestamp.field contexts metrics].all? do |field|
|
325
|
-
@manifold_config&.dig(*field.split("."))
|
326
|
-
end
|
327
|
-
end
|
328
333
|
end
|
329
334
|
end
|
330
335
|
end
|
data/lib/manifold/version.rb
CHANGED
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: manifold-cli
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.17
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- claytongentry
|
8
8
|
bindir: bin
|
9
9
|
cert_chain: []
|
10
|
-
date: 2025-02-
|
10
|
+
date: 2025-02-18 00:00:00.000000000 Z
|
11
11
|
dependencies:
|
12
12
|
- !ruby/object:Gem::Dependency
|
13
13
|
name: thor
|