manifold-cli 0.0.17 → 0.0.18
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 26f9112132a14f5bbb3cd123dc4d399ed9d618ec41638eaf43b5bbb0f870b6dd
|
4
|
+
data.tar.gz: cc8bc857ca5163f2e86ffe8656fe48b8fb46c3295bfa7f3ffad531597e0b15df
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: dba055e83e3ef141fd49c8af5d7079d33a81ac6a54539c32270e86ecbd73840ee183f611e99f7c985c1b7c5cbb7bd32b2ac650abc8faa85946600a2ddf10af1b
|
7
|
+
data.tar.gz: d44010ad67f25f6ee9bd4762ee34a319062ebb82d68af81192bdba8becebad6850fd65e3e5e12f628114dd19ba17378bbecf196320e226613bb29e4062b3a80d
|
@@ -172,11 +172,7 @@ module Manifold
|
|
172
172
|
return unless manifold_file
|
173
173
|
|
174
174
|
sql_builder = Terraform::SQLBuilder.new(name, manifold_yaml)
|
175
|
-
|
176
|
-
sql = sql_builder.build_manifold_merge_sql(metrics_builder) do
|
177
|
-
metrics_builder.build_metrics_struct
|
178
|
-
end
|
179
|
-
|
175
|
+
sql = sql_builder.build_manifold_merge_sql
|
180
176
|
routines_directory.join("merge_manifold.sql").write(sql)
|
181
177
|
end
|
182
178
|
|
@@ -24,19 +24,5 @@ metrics:
|
|
24
24
|
sequenceSum:
|
25
25
|
field: context.sequence
|
26
26
|
|
27
|
-
source: my_project.
|
28
|
-
filter: timestamp >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 90 DAY)
|
29
|
-
|
30
|
-
taps:
|
31
|
-
breakouts:
|
32
|
-
paid: IS_PAID(context.location)
|
33
|
-
organic: IS_ORGANIC(context.location)
|
34
|
-
|
35
|
-
aggregations:
|
36
|
-
countif: tapCount
|
37
|
-
sumif:
|
38
|
-
sequenceSum:
|
39
|
-
field: context.sequence
|
40
|
-
|
41
|
-
source: my_project.my_dataset.my_table
|
27
|
+
source: my_project.render_metrics
|
42
28
|
filter: timestamp >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL 90 DAY)
|
@@ -3,110 +3,40 @@
|
|
3
3
|
module Manifold
|
4
4
|
module Terraform
|
5
5
|
# Handles building metrics SQL for manifold routines
|
6
|
-
class
|
7
|
-
def initialize(manifold_config)
|
6
|
+
class MetricsSQLBuilder
|
7
|
+
def initialize(name, manifold_config)
|
8
|
+
@name = name
|
8
9
|
@manifold_config = manifold_config
|
9
10
|
end
|
10
11
|
|
11
|
-
def
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
12
|
+
def build_metrics_select
|
13
|
+
<<~SQL
|
14
|
+
SELECT
|
15
|
+
id,
|
16
|
+
timestamp,
|
17
|
+
#{build_metrics_struct}
|
18
|
+
FROM #{build_metric_joins}
|
19
|
+
SQL
|
19
20
|
end
|
20
21
|
|
21
22
|
private
|
22
23
|
|
23
|
-
def
|
24
|
-
|
25
|
-
|
26
|
-
breakout_structs = build_breakout_structs(group_config)
|
27
|
-
return "" if breakout_structs.empty?
|
28
|
-
|
29
|
-
"\tSTRUCT(\n#{breakout_structs.join(",\n")}\n\t) AS #{group_name}"
|
30
|
-
end
|
31
|
-
|
32
|
-
def valid_group_config?(group_config)
|
33
|
-
group_config["breakouts"] &&
|
34
|
-
group_config["aggregations"] &&
|
35
|
-
!group_config["breakouts"].empty? &&
|
36
|
-
!group_config["aggregations"].empty?
|
37
|
-
end
|
38
|
-
|
39
|
-
def build_breakout_structs(group_config)
|
40
|
-
group_config["breakouts"].map do |name, config|
|
41
|
-
build_breakout_struct(name, config, group_config)
|
42
|
-
end.compact
|
43
|
-
end
|
44
|
-
|
45
|
-
def build_breakout_struct(name, config, group_config)
|
46
|
-
condition = build_breakout_condition(name, config, group_config)
|
47
|
-
metrics = build_breakout_metrics(group_config, condition)
|
48
|
-
return if metrics.empty?
|
49
|
-
|
50
|
-
"\t\tSTRUCT(\n\t\t\t#{metrics}\n\t\t) AS #{name}"
|
51
|
-
end
|
52
|
-
|
53
|
-
def build_breakout_metrics(group_config, condition)
|
54
|
-
metrics = []
|
55
|
-
add_count_metrics(metrics, group_config, condition)
|
56
|
-
add_sum_metrics(metrics, group_config, condition)
|
57
|
-
metrics.join(",\n\t\t\t")
|
58
|
-
end
|
59
|
-
|
60
|
-
def add_count_metrics(metrics, group_config, condition)
|
61
|
-
return unless group_config.dig("aggregations", "countif")
|
62
|
-
|
63
|
-
metrics << "COUNTIF(#{condition}) AS #{group_config["aggregations"]["countif"]}"
|
64
|
-
end
|
65
|
-
|
66
|
-
def add_sum_metrics(metrics, group_config, condition)
|
67
|
-
group_config.dig("aggregations", "sumif")&.each do |name, config|
|
68
|
-
metrics << "SUM(IF(#{condition}, #{config["field"]}, 0)) AS #{name}"
|
69
|
-
end
|
70
|
-
end
|
71
|
-
|
72
|
-
def build_breakout_condition(_name, config, group_config)
|
73
|
-
return config unless config.is_a?(Hash)
|
74
|
-
|
75
|
-
operator = config["operator"]
|
76
|
-
fields = config["fields"]
|
77
|
-
build_operator_condition(operator, fields, group_config)
|
78
|
-
end
|
79
|
-
|
80
|
-
def build_operator_condition(operator, fields, group_config)
|
81
|
-
conditions = fields.map { |f| group_config["breakouts"][f] }
|
82
|
-
case operator
|
83
|
-
when "AND", "OR" then join_conditions(conditions, operator)
|
84
|
-
when "NOT" then negate_condition(conditions.first)
|
85
|
-
when "NAND", "NOR" then negate_joined_conditions(conditions, operator[1..])
|
86
|
-
when "XOR" then build_xor_condition(conditions)
|
87
|
-
when "XNOR" then build_xnor_condition(conditions)
|
88
|
-
else config
|
89
|
-
end
|
90
|
-
end
|
91
|
-
|
92
|
-
def join_conditions(conditions, operator)
|
93
|
-
conditions.join(" #{operator} ")
|
94
|
-
end
|
95
|
-
|
96
|
-
def negate_condition(condition)
|
97
|
-
"NOT (#{condition})"
|
24
|
+
def build_metrics_struct
|
25
|
+
metric_groups = @manifold_config["metrics"].keys
|
26
|
+
metric_groups.map { |group| "#{group}.metrics #{group}" }.join(",\n ")
|
98
27
|
end
|
99
28
|
|
100
|
-
def
|
101
|
-
|
102
|
-
|
29
|
+
def build_metric_joins
|
30
|
+
metric_groups = @manifold_config["metrics"]
|
31
|
+
joins = metric_groups.map { |group, config| "#{config["source"]} AS #{group}" }
|
32
|
+
first = joins.shift
|
33
|
+
return first if joins.empty?
|
103
34
|
|
104
|
-
|
105
|
-
"(#{conditions[0]} AND NOT #{conditions[1]}) OR (NOT #{conditions[0]} AND #{conditions[1]})"
|
35
|
+
"#{first}\n #{joins.map { |table| "FULL OUTER JOIN #{table} USING (id, timestamp)" }.join("\n ")}"
|
106
36
|
end
|
107
37
|
|
108
|
-
def
|
109
|
-
"
|
38
|
+
def timestamp_field
|
39
|
+
@manifold_config&.dig("timestamp", "field")
|
110
40
|
end
|
111
41
|
end
|
112
42
|
|
@@ -115,17 +45,17 @@ module Manifold
|
|
115
45
|
def initialize(name, manifold_config)
|
116
46
|
@name = name
|
117
47
|
@manifold_config = manifold_config
|
48
|
+
@metrics_builder = MetricsSQLBuilder.new(name, manifold_config)
|
118
49
|
end
|
119
50
|
|
120
|
-
def build_manifold_merge_sql
|
51
|
+
def build_manifold_merge_sql
|
121
52
|
return "" unless valid_config?
|
122
53
|
|
123
54
|
<<~SQL
|
124
55
|
MERGE #{@name}.Manifold AS target USING (
|
125
|
-
#{
|
126
|
-
#{build_final_select}
|
56
|
+
#{build_source_query}
|
127
57
|
) AS source
|
128
|
-
|
58
|
+
#{build_merge_conditions}
|
129
59
|
#{build_merge_actions}
|
130
60
|
SQL
|
131
61
|
end
|
@@ -153,55 +83,30 @@ module Manifold
|
|
153
83
|
first_group&.dig("source")
|
154
84
|
end
|
155
85
|
|
156
|
-
def interval
|
157
|
-
@manifold_config&.dig("timestamp", "interval") || "DAY"
|
158
|
-
end
|
159
|
-
|
160
|
-
def where_clause
|
161
|
-
first_group = @manifold_config["metrics"]&.values&.first
|
162
|
-
return "" unless first_group&.dig("filter")
|
163
|
-
|
164
|
-
"WHERE #{first_group["filter"]}"
|
165
|
-
end
|
166
|
-
|
167
86
|
def timestamp_field
|
168
87
|
@manifold_config&.dig("timestamp", "field")
|
169
88
|
end
|
170
89
|
|
171
|
-
def
|
90
|
+
def build_source_query
|
172
91
|
<<~SQL
|
173
92
|
WITH Metrics AS (
|
174
|
-
#{build_metrics_select
|
93
|
+
#{@metrics_builder.build_metrics_select}
|
175
94
|
)
|
176
|
-
SQL
|
177
|
-
end
|
178
95
|
|
179
|
-
def build_metrics_select(&block)
|
180
|
-
<<~SQL
|
181
|
-
SELECT
|
182
|
-
id,
|
183
|
-
TIMESTAMP_TRUNC(#{timestamp_field}, #{interval}) timestamp,
|
184
|
-
STRUCT(
|
185
|
-
#{block.call}
|
186
|
-
) AS metrics
|
187
|
-
FROM #{source_table}
|
188
|
-
#{where_clause}
|
189
|
-
GROUP BY 1, 2
|
190
|
-
SQL
|
191
|
-
end
|
192
|
-
|
193
|
-
def build_final_select
|
194
|
-
<<~SQL
|
195
96
|
SELECT
|
196
97
|
id,
|
197
98
|
timestamp,
|
198
99
|
Dimensions.dimensions,
|
199
|
-
Metrics
|
100
|
+
(SELECT AS STRUCT Metrics.* EXCEPT(id, timestamp)) metrics
|
200
101
|
FROM Metrics
|
201
|
-
|
102
|
+
JOIN #{@name}.Dimensions USING (id)
|
202
103
|
SQL
|
203
104
|
end
|
204
105
|
|
106
|
+
def build_merge_conditions
|
107
|
+
"ON source.id = target.id AND source.timestamp = target.timestamp"
|
108
|
+
end
|
109
|
+
|
205
110
|
def build_merge_actions
|
206
111
|
<<~SQL
|
207
112
|
WHEN MATCHED THEN
|
@@ -298,8 +203,8 @@ module Manifold
|
|
298
203
|
|
299
204
|
def routine_config
|
300
205
|
routines = {
|
301
|
-
"merge_dimensions" => dimensions_routine_attributes
|
302
|
-
|
206
|
+
"merge_dimensions" => dimensions_routine_attributes,
|
207
|
+
"merge_manifold" => manifold_routine_attributes
|
303
208
|
}.compact
|
304
209
|
|
305
210
|
routines.empty? ? nil : routines
|
data/lib/manifold/version.rb
CHANGED
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: manifold-cli
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.18
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- claytongentry
|
8
8
|
bindir: bin
|
9
9
|
cert_chain: []
|
10
|
-
date: 2025-02-
|
10
|
+
date: 2025-02-19 00:00:00.000000000 Z
|
11
11
|
dependencies:
|
12
12
|
- !ruby/object:Gem::Dependency
|
13
13
|
name: thor
|