masamune 0.16.0 → 0.17.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/masamune/schema/dimension.rb +21 -1
- data/lib/masamune/transform/postgres/deduplicate_dimension.psql.erb +3 -2
- data/lib/masamune/transform/postgres/deduplicate_dimension.rb +21 -4
- data/lib/masamune/version.rb +1 -1
- data/spec/masamune/schema/dimension_spec.rb +20 -0
- data/spec/masamune/transform/deduplicate_dimension_spec.rb +6 -5
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fed58d83f573640ea93aaee3226ba39e4b504302
|
4
|
+
data.tar.gz: a40fb42e8f9dd313c7606266424df457b61b5cef
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1eef004f660f92fdb0664e65df4f402d0cf68059af223140fb3f33d5ad2a4c9a060543555d1cab548b46f4971e0fddc9ec4f1f2f48ee964ad596cc99dd6386c8
|
7
|
+
data.tar.gz: c07cd4e970b1208efcff02c052538590c9f8d27dbd53efaeedcd9845e7a19e37505954c2360d5835fb7588505e089378c7538549c4ada9cfe6fdcbc82a8bdd4d
|
@@ -22,11 +22,31 @@
|
|
22
22
|
|
23
23
|
module Masamune::Schema
|
24
24
|
class Dimension < Table
|
25
|
+
SUPPORTED_GRAINS = [:hourly, :daily, :monthly]
|
26
|
+
|
25
27
|
def initialize(opts = {})
|
26
|
-
|
28
|
+
opts.symbolize_keys!
|
29
|
+
self.grain = opts.delete(:grain)
|
30
|
+
super opts
|
27
31
|
initialize_dimension_columns!
|
28
32
|
end
|
29
33
|
|
34
|
+
def grain
|
35
|
+
return @grain if @grain
|
36
|
+
case type
|
37
|
+
when :date
|
38
|
+
:daily
|
39
|
+
else
|
40
|
+
:hourly
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def grain=(grain = nil)
|
45
|
+
return unless grain
|
46
|
+
raise ArgumentError, "unknown grain '#{grain}'" unless SUPPORTED_GRAINS.include?(grain.to_sym)
|
47
|
+
@grain = grain.to_sym
|
48
|
+
end
|
49
|
+
|
30
50
|
def suffix
|
31
51
|
suffix = case type
|
32
52
|
when :mini
|
@@ -22,12 +22,13 @@
|
|
22
22
|
|
23
23
|
WITH consolidated AS (
|
24
24
|
SELECT
|
25
|
-
<%- target.
|
25
|
+
<%- target.insert_view_last_values('w').each do |value| -%>
|
26
26
|
<%= value %><%= ',' %>
|
27
27
|
<%- end -%>
|
28
|
-
start_at
|
28
|
+
<%= target.start_at_with_grain %> AS start_at
|
29
29
|
FROM
|
30
30
|
<%= source.name %>
|
31
|
+
WINDOW w AS (PARTITION BY <%= target.window(target.start_at_with_grain).join(', ') %> ORDER BY start_at ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)
|
31
32
|
)
|
32
33
|
INSERT INTO
|
33
34
|
<%= target.name %> (<%= target.insert_columns.join(', ') %>, start_at)
|
@@ -42,12 +42,18 @@ module Masamune::Transform::Postgres
|
|
42
42
|
consolidated_columns.map { |_, column| column.name }
|
43
43
|
end
|
44
44
|
|
45
|
-
def insert_view_values
|
45
|
+
def insert_view_values
|
46
46
|
consolidated_columns.map do |_, column|
|
47
|
-
|
48
|
-
|
47
|
+
column.name
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def insert_view_last_values(window)
|
52
|
+
consolidated_columns.map do |_, column|
|
53
|
+
if column.default.nil?
|
54
|
+
"LAST_VALUE(#{column.name}) OVER #{window} AS #{column.name}"
|
49
55
|
else
|
50
|
-
column.name
|
56
|
+
"COALESCE(LAST_VALUE(#{column.name}) OVER #{window}, #{column.sql_value(column.default)}) AS #{column.name}"
|
51
57
|
end
|
52
58
|
end
|
53
59
|
end
|
@@ -68,6 +74,17 @@ module Masamune::Transform::Postgres
|
|
68
74
|
(columns.values.select { |column| extra.delete(column.name) || column.natural_key || column.auto_reference }.map(&:name) + extra).uniq
|
69
75
|
end
|
70
76
|
|
77
|
+
def start_at_with_grain
|
78
|
+
case grain
|
79
|
+
when :hourly
|
80
|
+
"date_trunc('hour', start_at)"
|
81
|
+
when :daily
|
82
|
+
"date_trunc('day', start_at)"
|
83
|
+
when :monthly
|
84
|
+
"date_trunc('month', start_at)"
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
71
88
|
private
|
72
89
|
|
73
90
|
def consolidated_columns
|
data/lib/masamune/version.rb
CHANGED
@@ -33,6 +33,7 @@ describe Masamune::Schema::Dimension do
|
|
33
33
|
|
34
34
|
it { expect(dimension.name).to eq('date_dimension') }
|
35
35
|
it { expect(dimension.type).to eq(:date) }
|
36
|
+
it { expect(dimension.grain).to eq(:daily) }
|
36
37
|
end
|
37
38
|
|
38
39
|
context 'for type :one' do
|
@@ -46,6 +47,7 @@ describe Masamune::Schema::Dimension do
|
|
46
47
|
|
47
48
|
it { expect(dimension.name).to eq('user_dimension') }
|
48
49
|
it { expect(dimension.type).to eq(:one) }
|
50
|
+
it { expect(dimension.grain).to eq(:hourly) }
|
49
51
|
end
|
50
52
|
|
51
53
|
context 'for type :two' do
|
@@ -59,6 +61,7 @@ describe Masamune::Schema::Dimension do
|
|
59
61
|
|
60
62
|
it { expect(dimension.name).to eq('user_dimension') }
|
61
63
|
it { expect(dimension.type).to eq(:two) }
|
64
|
+
it { expect(dimension.grain).to eq(:hourly) }
|
62
65
|
end
|
63
66
|
|
64
67
|
context 'with invalid values' do
|
@@ -106,6 +109,7 @@ describe Masamune::Schema::Dimension do
|
|
106
109
|
|
107
110
|
it { expect(dimension.name).to eq('user_dimension') }
|
108
111
|
it { expect(dimension.type).to eq(:four) }
|
112
|
+
it { expect(dimension.grain).to eq(:hourly) }
|
109
113
|
|
110
114
|
describe '#stage_table' do
|
111
115
|
let!(:stage_table) { dimension.stage_table }
|
@@ -132,4 +136,20 @@ describe Masamune::Schema::Dimension do
|
|
132
136
|
end
|
133
137
|
end
|
134
138
|
end
|
139
|
+
|
140
|
+
context 'dimension with daily grain' do
|
141
|
+
let(:dimension) { described_class.new id: 'users', type: :one, grain: :daily }
|
142
|
+
|
143
|
+
it { expect(dimension.name).to eq('users_dimension') }
|
144
|
+
it { expect(dimension.type).to eq(:one) }
|
145
|
+
it { expect(dimension.grain).to eq(:daily) }
|
146
|
+
end
|
147
|
+
|
148
|
+
context 'dimension with unknown grain' do
|
149
|
+
subject(:dimension) do
|
150
|
+
described_class.new id: 'users', grain: :quarterly
|
151
|
+
end
|
152
|
+
|
153
|
+
it { expect { dimension }.to raise_error ArgumentError, "unknown grain 'quarterly'" }
|
154
|
+
end
|
135
155
|
end
|
@@ -46,13 +46,14 @@ describe Masamune::Transform::DeduplicateDimension do
|
|
46
46
|
is_expected.to eq <<-EOS.strip_heredoc
|
47
47
|
WITH consolidated AS (
|
48
48
|
SELECT
|
49
|
-
user_account_state_type_id,
|
50
|
-
tenant_id,
|
51
|
-
user_id,
|
52
|
-
preferences,
|
53
|
-
start_at
|
49
|
+
LAST_VALUE(user_account_state_type_id) OVER w AS user_account_state_type_id,
|
50
|
+
LAST_VALUE(tenant_id) OVER w AS tenant_id,
|
51
|
+
LAST_VALUE(user_id) OVER w AS user_id,
|
52
|
+
LAST_VALUE(preferences) OVER w AS preferences,
|
53
|
+
date_trunc('hour', start_at) AS start_at
|
54
54
|
FROM
|
55
55
|
user_consolidated_dimension_stage
|
56
|
+
WINDOW w AS (PARTITION BY tenant_id, user_id, date_trunc('hour', start_at) ORDER BY start_at ROWS BETWEEN UNBOUNDED PRECEDING AND UNBOUNDED FOLLOWING)
|
56
57
|
)
|
57
58
|
INSERT INTO
|
58
59
|
user_deduplicated_dimension_stage (user_account_state_type_id, tenant_id, user_id, preferences, start_at)
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: masamune
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.17.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Michael Andrews
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-01-
|
11
|
+
date: 2016-01-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: thor
|