manifold-cli 0.0.16 → 0.0.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/manifold/api/schema_generator.rb +26 -21
- data/lib/manifold/api/workspace.rb +112 -36
- data/lib/manifold/cli.rb +1 -1
- data/lib/manifold/templates/workspace_template.yml +12 -15
- data/lib/manifold/terraform/workspace_configuration.rb +51 -141
- data/lib/manifold/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 26f9112132a14f5bbb3cd123dc4d399ed9d618ec41638eaf43b5bbb0f870b6dd
|
4
|
+
data.tar.gz: cc8bc857ca5163f2e86ffe8656fe48b8fb46c3295bfa7f3ffad531597e0b15df
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: dba055e83e3ef141fd49c8af5d7079d33a81ac6a54539c32270e86ecbd73840ee183f611e99f7c985c1b7c5cbb7bd32b2ac650abc8faa85946600a2ddf10af1b
|
7
|
+
data.tar.gz: d44010ad67f25f6ee9bd4762ee34a319062ebb82d68af81192bdba8becebad6850fd65e3e5e12f628114dd19ba17378bbecf196320e226613bb29e4062b3a80d
|
@@ -4,8 +4,6 @@ module Manifold
|
|
4
4
|
module API
|
5
5
|
# Handles schema generation for Manifold tables
|
6
6
|
class SchemaGenerator
|
7
|
-
VALID_OPERATORS = %w[AND OR NOT NAND NOR XOR XNOR].freeze
|
8
|
-
|
9
7
|
def initialize(dimensions_fields, manifold_yaml)
|
10
8
|
@dimensions_fields = dimensions_fields
|
11
9
|
@manifold_yaml = manifold_yaml
|
@@ -33,39 +31,52 @@ module Manifold
|
|
33
31
|
private
|
34
32
|
|
35
33
|
def metrics_fields
|
36
|
-
return [] unless @manifold_yaml["
|
34
|
+
return [] unless @manifold_yaml["metrics"]
|
35
|
+
|
36
|
+
@manifold_yaml["metrics"].map do |group_name, group_config|
|
37
|
+
{
|
38
|
+
"name" => group_name,
|
39
|
+
"type" => "RECORD",
|
40
|
+
"mode" => "NULLABLE",
|
41
|
+
"fields" => group_metrics_fields(group_config)
|
42
|
+
}
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def group_metrics_fields(group_config)
|
47
|
+
return [] unless group_config["breakouts"] && group_config["aggregations"]
|
37
48
|
|
38
|
-
|
49
|
+
group_config["breakouts"].map do |breakout_name, _breakout_config|
|
39
50
|
{
|
40
|
-
"name" =>
|
51
|
+
"name" => breakout_name,
|
41
52
|
"type" => "RECORD",
|
42
53
|
"mode" => "NULLABLE",
|
43
|
-
"fields" =>
|
54
|
+
"fields" => breakout_metrics_fields(group_config)
|
44
55
|
}
|
45
56
|
end
|
46
57
|
end
|
47
58
|
|
48
|
-
def
|
59
|
+
def breakout_metrics_fields(group_config)
|
49
60
|
[
|
50
|
-
*countif_fields,
|
51
|
-
*sumif_fields
|
61
|
+
*countif_fields(group_config),
|
62
|
+
*sumif_fields(group_config)
|
52
63
|
]
|
53
64
|
end
|
54
65
|
|
55
|
-
def countif_fields
|
56
|
-
return [] unless
|
66
|
+
def countif_fields(group_config)
|
67
|
+
return [] unless group_config.dig("aggregations", "countif")
|
57
68
|
|
58
69
|
[{
|
59
|
-
"name" =>
|
70
|
+
"name" => group_config["aggregations"]["countif"],
|
60
71
|
"type" => "INTEGER",
|
61
72
|
"mode" => "NULLABLE"
|
62
73
|
}]
|
63
74
|
end
|
64
75
|
|
65
|
-
def sumif_fields
|
66
|
-
return [] unless
|
76
|
+
def sumif_fields(group_config)
|
77
|
+
return [] unless group_config.dig("aggregations", "sumif")
|
67
78
|
|
68
|
-
|
79
|
+
group_config["aggregations"]["sumif"].keys.map do |metric_name|
|
69
80
|
{
|
70
81
|
"name" => metric_name,
|
71
82
|
"type" => "INTEGER",
|
@@ -73,12 +84,6 @@ module Manifold
|
|
73
84
|
}
|
74
85
|
end
|
75
86
|
end
|
76
|
-
|
77
|
-
def validate_operator!(operator)
|
78
|
-
return if VALID_OPERATORS.include?(operator)
|
79
|
-
|
80
|
-
raise ArgumentError, "Invalid operator: #{operator}. Valid operators are: #{VALID_OPERATORS.join(", ")}"
|
81
|
-
end
|
82
87
|
end
|
83
88
|
end
|
84
89
|
end
|
@@ -19,12 +19,91 @@ module Manifold
|
|
19
19
|
vector_config = @vector_service.load_vector_config(vector)
|
20
20
|
config.add_vector(vector_config)
|
21
21
|
end
|
22
|
-
config.
|
22
|
+
config.dimensions_config = @manifold_yaml["dimensions"]&.fetch("merge", nil) if @manifold_yaml["dimensions"]
|
23
23
|
config.manifold_config = @manifold_yaml
|
24
24
|
config.write(path)
|
25
25
|
end
|
26
26
|
end
|
27
27
|
|
28
|
+
# Handles SQL generation for manifold workspaces
|
29
|
+
class SqlGenerator
|
30
|
+
def initialize(name, manifold_yaml)
|
31
|
+
@name = name
|
32
|
+
@manifold_yaml = manifold_yaml
|
33
|
+
end
|
34
|
+
|
35
|
+
def generate_dimensions_merge_sql(source_sql)
|
36
|
+
return unless valid_dimensions_config?
|
37
|
+
|
38
|
+
sql_builder = Terraform::SQLBuilder.new(@name, @manifold_yaml)
|
39
|
+
sql_builder.build_dimensions_merge_sql(source_sql)
|
40
|
+
end
|
41
|
+
|
42
|
+
private
|
43
|
+
|
44
|
+
def valid_dimensions_config?
|
45
|
+
return false unless @manifold_yaml
|
46
|
+
|
47
|
+
!@manifold_yaml["dimensions"]&.dig("merge", "source").nil?
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
# Handles schema file generation for manifold workspaces
|
52
|
+
class SchemaWriter
|
53
|
+
def initialize(name, vectors, vector_service, manifold_yaml, logger)
|
54
|
+
@name = name
|
55
|
+
@vectors = vectors
|
56
|
+
@vector_service = vector_service
|
57
|
+
@manifold_yaml = manifold_yaml
|
58
|
+
@logger = logger
|
59
|
+
end
|
60
|
+
|
61
|
+
def write_schemas(tables_directory)
|
62
|
+
tables_directory.mkpath
|
63
|
+
write_dimensions_schema(tables_directory)
|
64
|
+
write_manifold_schema(tables_directory)
|
65
|
+
end
|
66
|
+
|
67
|
+
private
|
68
|
+
|
69
|
+
def write_dimensions_schema(tables_directory)
|
70
|
+
dimensions_path = tables_directory.join("dimensions.json")
|
71
|
+
dimensions_path.write(dimensions_schema_json.concat("\n"))
|
72
|
+
end
|
73
|
+
|
74
|
+
def write_manifold_schema(tables_directory)
|
75
|
+
manifold_path = tables_directory.join("manifold.json")
|
76
|
+
manifold_path.write(manifold_schema_json.concat("\n"))
|
77
|
+
end
|
78
|
+
|
79
|
+
def schema_generator
|
80
|
+
@schema_generator ||= SchemaGenerator.new(dimensions_fields, @manifold_yaml)
|
81
|
+
end
|
82
|
+
|
83
|
+
def manifold_schema
|
84
|
+
schema_generator.manifold_schema
|
85
|
+
end
|
86
|
+
|
87
|
+
def dimensions_schema
|
88
|
+
schema_generator.dimensions_schema
|
89
|
+
end
|
90
|
+
|
91
|
+
def dimensions_fields
|
92
|
+
@dimensions_fields ||= @vectors.filter_map do |vector|
|
93
|
+
@logger.info("Loading vector schema for '#{vector}'.")
|
94
|
+
@vector_service.load_vector_schema(vector)
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
def dimensions_schema_json
|
99
|
+
JSON.pretty_generate(dimensions_schema)
|
100
|
+
end
|
101
|
+
|
102
|
+
def manifold_schema_json
|
103
|
+
JSON.pretty_generate(manifold_schema)
|
104
|
+
end
|
105
|
+
end
|
106
|
+
|
28
107
|
# Encapsulates a single manifold.
|
29
108
|
class Workspace
|
30
109
|
attr_reader :name, :template_path, :logger
|
@@ -52,13 +131,13 @@ module Manifold
|
|
52
131
|
def generate(with_terraform: false)
|
53
132
|
return nil unless manifold_exists? && any_vectors?
|
54
133
|
|
55
|
-
|
56
|
-
generate_dimensions
|
57
|
-
generate_manifold
|
134
|
+
generate_schemas
|
58
135
|
logger.info("Generated BigQuery dimensions table schema for workspace '#{name}'.")
|
59
136
|
|
60
137
|
return unless with_terraform
|
61
138
|
|
139
|
+
write_manifold_merge_sql
|
140
|
+
write_dimensions_merge_sql
|
62
141
|
generate_terraform
|
63
142
|
logger.info("Generated Terraform configuration for workspace '#{name}'.")
|
64
143
|
end
|
@@ -89,53 +168,54 @@ module Manifold
|
|
89
168
|
directory.join("main.tf.json")
|
90
169
|
end
|
91
170
|
|
92
|
-
|
171
|
+
def write_manifold_merge_sql
|
172
|
+
return unless manifold_file
|
93
173
|
|
94
|
-
|
95
|
-
|
174
|
+
sql_builder = Terraform::SQLBuilder.new(name, manifold_yaml)
|
175
|
+
sql = sql_builder.build_manifold_merge_sql
|
176
|
+
routines_directory.join("merge_manifold.sql").write(sql)
|
96
177
|
end
|
97
178
|
|
98
|
-
def
|
99
|
-
|
100
|
-
end
|
179
|
+
def write_dimensions_merge_sql
|
180
|
+
return unless dimensions_merge_source_exists?
|
101
181
|
|
102
|
-
|
103
|
-
|
104
|
-
end
|
182
|
+
sql = generate_dimensions_merge_sql
|
183
|
+
return unless sql
|
105
184
|
|
106
|
-
|
107
|
-
manifold_schema_path.write(manifold_schema_json.concat("\n"))
|
185
|
+
write_dimensions_merge_sql_file(sql)
|
108
186
|
end
|
109
187
|
|
110
|
-
def
|
111
|
-
|
188
|
+
def dimensions_merge_source_exists?
|
189
|
+
manifold_yaml["dimensions"]&.dig("merge", "source")
|
112
190
|
end
|
113
191
|
|
114
|
-
def
|
115
|
-
|
192
|
+
def generate_dimensions_merge_sql
|
193
|
+
source_sql = File.read(Pathname.pwd.join(manifold_yaml["dimensions"]["merge"]["source"]))
|
194
|
+
SqlGenerator.new(name, manifold_yaml).generate_dimensions_merge_sql(source_sql)
|
116
195
|
end
|
117
196
|
|
118
|
-
def
|
119
|
-
|
197
|
+
def write_dimensions_merge_sql_file(sql)
|
198
|
+
routines_directory.mkpath
|
199
|
+
dimensions_merge_sql_path.write(sql)
|
120
200
|
end
|
121
201
|
|
122
|
-
def
|
123
|
-
|
202
|
+
def dimensions_merge_sql_path
|
203
|
+
routines_directory.join("merge_dimensions.sql")
|
124
204
|
end
|
125
205
|
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
end
|
206
|
+
private
|
207
|
+
|
208
|
+
def directory
|
209
|
+
Pathname.pwd.join("workspaces", name)
|
131
210
|
end
|
132
211
|
|
133
|
-
def
|
134
|
-
|
212
|
+
def manifold_yaml
|
213
|
+
@manifold_yaml ||= YAML.safe_load_file(manifold_path)
|
135
214
|
end
|
136
215
|
|
137
|
-
def
|
138
|
-
|
216
|
+
def generate_schemas
|
217
|
+
SchemaWriter.new(name, vectors, @vector_service, manifold_yaml, logger)
|
218
|
+
.write_schemas(tables_directory)
|
139
219
|
end
|
140
220
|
|
141
221
|
def any_vectors?
|
@@ -151,10 +231,6 @@ module Manifold
|
|
151
231
|
terraform_generator.manifold_config = manifold_yaml
|
152
232
|
terraform_generator.generate(terraform_main_path)
|
153
233
|
end
|
154
|
-
|
155
|
-
def manifold_schema_json
|
156
|
-
JSON.pretty_generate(manifold_schema)
|
157
|
-
end
|
158
234
|
end
|
159
235
|
end
|
160
236
|
end
|
data/lib/manifold/cli.rb
CHANGED
@@ -4,7 +4,7 @@ module Manifold
|
|
4
4
|
# CLI provides command line interface functionality
|
5
5
|
# for creating and managing umbrella projects for data management.
|
6
6
|
class CLI < Thor
|
7
|
-
attr_accessor :logger
|
7
|
+
attr_accessor :logger
|
8
8
|
|
9
9
|
def initialize(*args, logger: Logger.new($stdout))
|
10
10
|
super(*args)
|
@@ -12,20 +12,17 @@ timestamp:
|
|
12
12
|
interval: HOUR
|
13
13
|
field: timestamp
|
14
14
|
|
15
|
-
contexts:
|
16
|
-
paid: IS_PAID(context.location)
|
17
|
-
organic: IS_ORGANIC(context.location)
|
18
|
-
paidOrganic:
|
19
|
-
fields:
|
20
|
-
- paid
|
21
|
-
- organic
|
22
|
-
operator: AND
|
23
|
-
|
24
15
|
metrics:
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
16
|
+
renders:
|
17
|
+
breakouts:
|
18
|
+
paid: IS_PAID(context.location)
|
19
|
+
organic: IS_ORGANIC(context.location)
|
20
|
+
|
21
|
+
aggregations:
|
22
|
+
countif: renderCount
|
23
|
+
sumif:
|
24
|
+
sequenceSum:
|
25
|
+
field: context.sequence
|
29
26
|
|
30
|
-
source: my_project.
|
31
|
-
filter: timestamp >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 90 DAY)
|
27
|
+
source: my_project.render_metrics
|
28
|
+
filter: timestamp >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 90 DAY)
|
@@ -3,82 +3,40 @@
|
|
3
3
|
module Manifold
|
4
4
|
module Terraform
|
5
5
|
# Handles building metrics SQL for manifold routines
|
6
|
-
class
|
7
|
-
def initialize(manifold_config)
|
6
|
+
class MetricsSQLBuilder
|
7
|
+
def initialize(name, manifold_config)
|
8
|
+
@name = name
|
8
9
|
@manifold_config = manifold_config
|
9
10
|
end
|
10
11
|
|
11
|
-
def
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
context_structs.join(",\n")
|
12
|
+
def build_metrics_select
|
13
|
+
<<~SQL
|
14
|
+
SELECT
|
15
|
+
id,
|
16
|
+
timestamp,
|
17
|
+
#{build_metrics_struct}
|
18
|
+
FROM #{build_metric_joins}
|
19
|
+
SQL
|
21
20
|
end
|
22
21
|
|
23
22
|
private
|
24
23
|
|
25
|
-
def
|
26
|
-
|
27
|
-
|
28
|
-
add_sum_metrics(metrics, condition)
|
29
|
-
metrics.join(",\n")
|
30
|
-
end
|
31
|
-
|
32
|
-
def add_count_metrics(metrics, condition)
|
33
|
-
return unless @manifold_config.dig("metrics", "countif")
|
34
|
-
|
35
|
-
metrics << "COUNTIF(#{condition}) AS #{@manifold_config["metrics"]["countif"]}"
|
36
|
-
end
|
37
|
-
|
38
|
-
def add_sum_metrics(metrics, condition)
|
39
|
-
@manifold_config.dig("metrics", "sumif")&.each do |name, config|
|
40
|
-
metrics << "SUM(IF(#{condition}, #{config["field"]}, 0)) AS #{name}"
|
41
|
-
end
|
42
|
-
end
|
43
|
-
|
44
|
-
def build_context_condition(_name, config)
|
45
|
-
return config unless config.is_a?(Hash)
|
46
|
-
|
47
|
-
operator = config["operator"]
|
48
|
-
fields = config["fields"]
|
49
|
-
build_operator_condition(operator, fields)
|
50
|
-
end
|
51
|
-
|
52
|
-
def build_operator_condition(operator, fields)
|
53
|
-
conditions = fields.map { |f| @manifold_config["contexts"][f] }
|
54
|
-
case operator
|
55
|
-
when "AND", "OR" then join_conditions(conditions, operator)
|
56
|
-
when "NOT" then negate_condition(conditions.first)
|
57
|
-
when "NAND", "NOR" then negate_joined_conditions(conditions, operator[1..])
|
58
|
-
when "XOR" then build_xor_condition(conditions)
|
59
|
-
when "XNOR" then build_xnor_condition(conditions)
|
60
|
-
else config
|
61
|
-
end
|
62
|
-
end
|
63
|
-
|
64
|
-
def join_conditions(conditions, operator)
|
65
|
-
conditions.join(" #{operator} ")
|
66
|
-
end
|
67
|
-
|
68
|
-
def negate_condition(condition)
|
69
|
-
"NOT (#{condition})"
|
24
|
+
def build_metrics_struct
|
25
|
+
metric_groups = @manifold_config["metrics"].keys
|
26
|
+
metric_groups.map { |group| "#{group}.metrics #{group}" }.join(",\n ")
|
70
27
|
end
|
71
28
|
|
72
|
-
def
|
73
|
-
|
74
|
-
|
29
|
+
def build_metric_joins
|
30
|
+
metric_groups = @manifold_config["metrics"]
|
31
|
+
joins = metric_groups.map { |group, config| "#{config["source"]} AS #{group}" }
|
32
|
+
first = joins.shift
|
33
|
+
return first if joins.empty?
|
75
34
|
|
76
|
-
|
77
|
-
"(#{conditions[0]} AND NOT #{conditions[1]}) OR (NOT #{conditions[0]} AND #{conditions[1]})"
|
35
|
+
"#{first}\n #{joins.map { |table| "FULL OUTER JOIN #{table} USING (id, timestamp)" }.join("\n ")}"
|
78
36
|
end
|
79
37
|
|
80
|
-
def
|
81
|
-
"
|
38
|
+
def timestamp_field
|
39
|
+
@manifold_config&.dig("timestamp", "field")
|
82
40
|
end
|
83
41
|
end
|
84
42
|
|
@@ -87,17 +45,17 @@ module Manifold
|
|
87
45
|
def initialize(name, manifold_config)
|
88
46
|
@name = name
|
89
47
|
@manifold_config = manifold_config
|
48
|
+
@metrics_builder = MetricsSQLBuilder.new(name, manifold_config)
|
90
49
|
end
|
91
50
|
|
92
|
-
def build_manifold_merge_sql
|
51
|
+
def build_manifold_merge_sql
|
93
52
|
return "" unless valid_config?
|
94
53
|
|
95
54
|
<<~SQL
|
96
55
|
MERGE #{@name}.Manifold AS target USING (
|
97
|
-
#{
|
98
|
-
#{build_final_select}
|
56
|
+
#{build_source_query}
|
99
57
|
) AS source
|
100
|
-
|
58
|
+
#{build_merge_conditions}
|
101
59
|
#{build_merge_actions}
|
102
60
|
SQL
|
103
61
|
end
|
@@ -117,37 +75,36 @@ module Manifold
|
|
117
75
|
private
|
118
76
|
|
119
77
|
def valid_config?
|
120
|
-
source_table && timestamp_field
|
78
|
+
source_table && timestamp_field && @manifold_config["metrics"]
|
79
|
+
end
|
80
|
+
|
81
|
+
def source_table
|
82
|
+
first_group = @manifold_config["metrics"]&.values&.first
|
83
|
+
first_group&.dig("source")
|
121
84
|
end
|
122
85
|
|
123
|
-
def
|
86
|
+
def timestamp_field
|
87
|
+
@manifold_config&.dig("timestamp", "field")
|
88
|
+
end
|
89
|
+
|
90
|
+
def build_source_query
|
124
91
|
<<~SQL
|
125
92
|
WITH Metrics AS (
|
126
|
-
#{build_metrics_select
|
93
|
+
#{@metrics_builder.build_metrics_select}
|
127
94
|
)
|
128
|
-
SQL
|
129
|
-
end
|
130
95
|
|
131
|
-
def build_metrics_select(&block)
|
132
|
-
<<~SQL
|
133
96
|
SELECT
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
138
|
-
|
139
|
-
|
140
|
-
#{where_clause}
|
141
|
-
GROUP BY 1, 2
|
97
|
+
id,
|
98
|
+
timestamp,
|
99
|
+
Dimensions.dimensions,
|
100
|
+
(SELECT AS STRUCT Metrics.* EXCEPT(id, timestamp)) metrics
|
101
|
+
FROM Metrics
|
102
|
+
JOIN #{@name}.Dimensions USING (id)
|
142
103
|
SQL
|
143
104
|
end
|
144
105
|
|
145
|
-
def
|
146
|
-
|
147
|
-
SELECT id, timestamp, #{@name}.Dimensions.dimensions, Metrics.metrics
|
148
|
-
FROM Metrics
|
149
|
-
LEFT JOIN #{@name}.Dimensions USING (id)
|
150
|
-
SQL
|
106
|
+
def build_merge_conditions
|
107
|
+
"ON source.id = target.id AND source.timestamp = target.timestamp"
|
151
108
|
end
|
152
109
|
|
153
110
|
def build_merge_actions
|
@@ -160,24 +117,6 @@ module Manifold
|
|
160
117
|
INSERT ROW;
|
161
118
|
SQL
|
162
119
|
end
|
163
|
-
|
164
|
-
def source_table
|
165
|
-
@manifold_config["source"]
|
166
|
-
end
|
167
|
-
|
168
|
-
def interval
|
169
|
-
@manifold_config&.dig("timestamp", "interval") || "DAY"
|
170
|
-
end
|
171
|
-
|
172
|
-
def where_clause
|
173
|
-
return "" unless @manifold_config["filter"]
|
174
|
-
|
175
|
-
"WHERE #{@manifold_config["filter"]}"
|
176
|
-
end
|
177
|
-
|
178
|
-
def timestamp_field
|
179
|
-
@manifold_config&.dig("timestamp", "field")
|
180
|
-
end
|
181
120
|
end
|
182
121
|
|
183
122
|
# Handles building table configurations
|
@@ -217,13 +156,13 @@ module Manifold
|
|
217
156
|
# Represents a Terraform configuration for a Manifold workspace.
|
218
157
|
class WorkspaceConfiguration < Configuration
|
219
158
|
attr_reader :name
|
220
|
-
attr_writer :
|
159
|
+
attr_writer :dimensions_config, :manifold_config
|
221
160
|
|
222
161
|
def initialize(name)
|
223
162
|
super()
|
224
163
|
@name = name
|
225
164
|
@vectors = []
|
226
|
-
@
|
165
|
+
@dimensions_config = nil
|
227
166
|
end
|
228
167
|
|
229
168
|
def add_vector(vector_config)
|
@@ -272,7 +211,7 @@ module Manifold
|
|
272
211
|
end
|
273
212
|
|
274
213
|
def dimensions_routine_attributes
|
275
|
-
return nil if @vectors.empty? || @
|
214
|
+
return nil if @vectors.empty? || @dimensions_config.nil?
|
276
215
|
|
277
216
|
{
|
278
217
|
"dataset_id" => name,
|
@@ -280,51 +219,22 @@ module Manifold
|
|
280
219
|
"routine_id" => "merge_dimensions",
|
281
220
|
"routine_type" => "PROCEDURE",
|
282
221
|
"language" => "SQL",
|
283
|
-
"definition_body" =>
|
222
|
+
"definition_body" => "${file(\"${path.module}/routines/merge_dimensions.sql\")}",
|
284
223
|
"depends_on" => ["google_bigquery_dataset.#{name}"]
|
285
224
|
}
|
286
225
|
end
|
287
226
|
|
288
|
-
def dimensions_merge_routine
|
289
|
-
return "" if @vectors.empty? || @merge_config.nil?
|
290
|
-
|
291
|
-
source_sql = File.read(Pathname.pwd.join(@merge_config["source"]))
|
292
|
-
SQLBuilder.new(name, @manifold_config).build_dimensions_merge_sql(source_sql)
|
293
|
-
end
|
294
|
-
|
295
227
|
def manifold_routine_attributes
|
296
|
-
return nil unless valid_manifold_config?
|
297
|
-
|
298
228
|
{
|
299
229
|
"dataset_id" => name,
|
300
230
|
"project" => "${var.project_id}",
|
301
231
|
"routine_id" => "merge_manifold",
|
302
232
|
"routine_type" => "PROCEDURE",
|
303
233
|
"language" => "SQL",
|
304
|
-
"definition_body" =>
|
234
|
+
"definition_body" => "${file(\"${path.module}/routines/merge_manifold.sql\")}",
|
305
235
|
"depends_on" => ["google_bigquery_dataset.#{name}"]
|
306
236
|
}
|
307
237
|
end
|
308
|
-
|
309
|
-
def manifold_merge_routine
|
310
|
-
metrics_builder = MetricsBuilder.new(@manifold_config)
|
311
|
-
sql_builder = SQLBuilder.new(name, @manifold_config)
|
312
|
-
sql_builder.build_manifold_merge_sql(metrics_builder) do
|
313
|
-
metrics_builder.build_metrics_struct
|
314
|
-
end
|
315
|
-
end
|
316
|
-
|
317
|
-
def valid_manifold_config?
|
318
|
-
return false unless @manifold_config
|
319
|
-
|
320
|
-
required_fields_present?
|
321
|
-
end
|
322
|
-
|
323
|
-
def required_fields_present?
|
324
|
-
%w[source timestamp.field contexts metrics].all? do |field|
|
325
|
-
@manifold_config&.dig(*field.split("."))
|
326
|
-
end
|
327
|
-
end
|
328
238
|
end
|
329
239
|
end
|
330
240
|
end
|
data/lib/manifold/version.rb
CHANGED
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: manifold-cli
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.18
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- claytongentry
|
8
8
|
bindir: bin
|
9
9
|
cert_chain: []
|
10
|
-
date: 2025-02-
|
10
|
+
date: 2025-02-19 00:00:00.000000000 Z
|
11
11
|
dependencies:
|
12
12
|
- !ruby/object:Gem::Dependency
|
13
13
|
name: thor
|