masamune 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/LICENSE.txt +21 -0
- data/README.md +54 -0
- data/Rakefile +15 -0
- data/bin/masamune-elastic-mapreduce +4 -0
- data/bin/masamune-hive +4 -0
- data/bin/masamune-psql +4 -0
- data/bin/masamune-shell +4 -0
- data/lib/masamune.rb +56 -0
- data/lib/masamune/accumulate.rb +60 -0
- data/lib/masamune/actions.rb +38 -0
- data/lib/masamune/actions/data_flow.rb +131 -0
- data/lib/masamune/actions/date_parse.rb +75 -0
- data/lib/masamune/actions/elastic_mapreduce.rb +68 -0
- data/lib/masamune/actions/execute.rb +52 -0
- data/lib/masamune/actions/filesystem.rb +37 -0
- data/lib/masamune/actions/hadoop_filesystem.rb +40 -0
- data/lib/masamune/actions/hadoop_streaming.rb +41 -0
- data/lib/masamune/actions/hive.rb +74 -0
- data/lib/masamune/actions/postgres.rb +76 -0
- data/lib/masamune/actions/postgres_admin.rb +34 -0
- data/lib/masamune/actions/s3cmd.rb +44 -0
- data/lib/masamune/actions/transform.rb +89 -0
- data/lib/masamune/after_initialize_callbacks.rb +55 -0
- data/lib/masamune/cached_filesystem.rb +110 -0
- data/lib/masamune/commands.rb +37 -0
- data/lib/masamune/commands/elastic_mapreduce.rb +119 -0
- data/lib/masamune/commands/hadoop_filesystem.rb +57 -0
- data/lib/masamune/commands/hadoop_streaming.rb +116 -0
- data/lib/masamune/commands/hive.rb +178 -0
- data/lib/masamune/commands/interactive.rb +37 -0
- data/lib/masamune/commands/postgres.rb +128 -0
- data/lib/masamune/commands/postgres_admin.rb +72 -0
- data/lib/masamune/commands/postgres_common.rb +33 -0
- data/lib/masamune/commands/retry_with_backoff.rb +60 -0
- data/lib/masamune/commands/s3cmd.rb +70 -0
- data/lib/masamune/commands/shell.rb +202 -0
- data/lib/masamune/configuration.rb +195 -0
- data/lib/masamune/data_plan.rb +31 -0
- data/lib/masamune/data_plan/builder.rb +66 -0
- data/lib/masamune/data_plan/elem.rb +190 -0
- data/lib/masamune/data_plan/engine.rb +162 -0
- data/lib/masamune/data_plan/rule.rb +292 -0
- data/lib/masamune/data_plan/set.rb +176 -0
- data/lib/masamune/environment.rb +164 -0
- data/lib/masamune/filesystem.rb +567 -0
- data/lib/masamune/has_environment.rb +40 -0
- data/lib/masamune/helpers.rb +27 -0
- data/lib/masamune/helpers/postgres.rb +84 -0
- data/lib/masamune/io.rb +33 -0
- data/lib/masamune/last_element.rb +53 -0
- data/lib/masamune/method_logger.rb +41 -0
- data/lib/masamune/multi_io.rb +39 -0
- data/lib/masamune/schema.rb +36 -0
- data/lib/masamune/schema/catalog.rb +233 -0
- data/lib/masamune/schema/column.rb +527 -0
- data/lib/masamune/schema/dimension.rb +133 -0
- data/lib/masamune/schema/event.rb +121 -0
- data/lib/masamune/schema/fact.rb +133 -0
- data/lib/masamune/schema/map.rb +265 -0
- data/lib/masamune/schema/row.rb +133 -0
- data/lib/masamune/schema/store.rb +115 -0
- data/lib/masamune/schema/table.rb +308 -0
- data/lib/masamune/schema/table_reference.rb +76 -0
- data/lib/masamune/spec_helper.rb +23 -0
- data/lib/masamune/string_format.rb +34 -0
- data/lib/masamune/tasks/elastic_mapreduce_thor.rb +60 -0
- data/lib/masamune/tasks/hive_thor.rb +55 -0
- data/lib/masamune/tasks/postgres_thor.rb +47 -0
- data/lib/masamune/tasks/shell_thor.rb +63 -0
- data/lib/masamune/template.rb +77 -0
- data/lib/masamune/thor.rb +186 -0
- data/lib/masamune/thor_loader.rb +38 -0
- data/lib/masamune/topological_hash.rb +34 -0
- data/lib/masamune/transform.rb +47 -0
- data/lib/masamune/transform/bulk_upsert.psql.erb +64 -0
- data/lib/masamune/transform/bulk_upsert.rb +52 -0
- data/lib/masamune/transform/consolidate_dimension.rb +54 -0
- data/lib/masamune/transform/deduplicate_dimension.psql.erb +52 -0
- data/lib/masamune/transform/deduplicate_dimension.rb +53 -0
- data/lib/masamune/transform/define_event_view.hql.erb +51 -0
- data/lib/masamune/transform/define_event_view.rb +60 -0
- data/lib/masamune/transform/define_index.psql.erb +34 -0
- data/lib/masamune/transform/define_schema.hql.erb +23 -0
- data/lib/masamune/transform/define_schema.psql.erb +79 -0
- data/lib/masamune/transform/define_schema.rb +56 -0
- data/lib/masamune/transform/define_table.hql.erb +34 -0
- data/lib/masamune/transform/define_table.psql.erb +95 -0
- data/lib/masamune/transform/define_table.rb +40 -0
- data/lib/masamune/transform/define_unique.psql.erb +30 -0
- data/lib/masamune/transform/insert_reference_values.psql.erb +43 -0
- data/lib/masamune/transform/insert_reference_values.rb +64 -0
- data/lib/masamune/transform/load_dimension.rb +47 -0
- data/lib/masamune/transform/load_fact.rb +45 -0
- data/lib/masamune/transform/operator.rb +96 -0
- data/lib/masamune/transform/relabel_dimension.psql.erb +76 -0
- data/lib/masamune/transform/relabel_dimension.rb +39 -0
- data/lib/masamune/transform/rollup_fact.psql.erb +79 -0
- data/lib/masamune/transform/rollup_fact.rb +149 -0
- data/lib/masamune/transform/snapshot_dimension.psql.erb +75 -0
- data/lib/masamune/transform/snapshot_dimension.rb +74 -0
- data/lib/masamune/transform/stage_dimension.psql.erb +39 -0
- data/lib/masamune/transform/stage_dimension.rb +83 -0
- data/lib/masamune/transform/stage_fact.psql.erb +80 -0
- data/lib/masamune/transform/stage_fact.rb +111 -0
- data/lib/masamune/version.rb +25 -0
- data/spec/fixtures/aggregate.sql.erb +25 -0
- data/spec/fixtures/comment.sql.erb +27 -0
- data/spec/fixtures/invalid.sql.erb +23 -0
- data/spec/fixtures/relative.sql.erb +23 -0
- data/spec/fixtures/simple.sql.erb +28 -0
- data/spec/fixtures/whitespace.sql.erb +30 -0
- data/spec/masamune/actions/elastic_mapreduce_spec.rb +108 -0
- data/spec/masamune/actions/execute_spec.rb +50 -0
- data/spec/masamune/actions/hadoop_filesystem_spec.rb +44 -0
- data/spec/masamune/actions/hadoop_streaming_spec.rb +74 -0
- data/spec/masamune/actions/hive_spec.rb +117 -0
- data/spec/masamune/actions/postgres_admin_spec.rb +58 -0
- data/spec/masamune/actions/postgres_spec.rb +134 -0
- data/spec/masamune/actions/s3cmd_spec.rb +44 -0
- data/spec/masamune/actions/transform_spec.rb +144 -0
- data/spec/masamune/after_initialization_callbacks_spec.rb +61 -0
- data/spec/masamune/cached_filesystem_spec.rb +167 -0
- data/spec/masamune/commands/hadoop_filesystem_spec.rb +50 -0
- data/spec/masamune/commands/hadoop_streaming_spec.rb +106 -0
- data/spec/masamune/commands/hive_spec.rb +117 -0
- data/spec/masamune/commands/postgres_admin_spec.rb +69 -0
- data/spec/masamune/commands/postgres_spec.rb +100 -0
- data/spec/masamune/commands/retry_with_backoff_spec.rb +116 -0
- data/spec/masamune/commands/s3cmd_spec.rb +50 -0
- data/spec/masamune/commands/shell_spec.rb +101 -0
- data/spec/masamune/configuration_spec.rb +102 -0
- data/spec/masamune/data_plan/builder_spec.rb +91 -0
- data/spec/masamune/data_plan/elem_spec.rb +102 -0
- data/spec/masamune/data_plan/engine_spec.rb +356 -0
- data/spec/masamune/data_plan/rule_spec.rb +407 -0
- data/spec/masamune/data_plan/set_spec.rb +517 -0
- data/spec/masamune/environment_spec.rb +65 -0
- data/spec/masamune/filesystem_spec.rb +1421 -0
- data/spec/masamune/helpers/postgres_spec.rb +95 -0
- data/spec/masamune/schema/catalog_spec.rb +613 -0
- data/spec/masamune/schema/column_spec.rb +696 -0
- data/spec/masamune/schema/dimension_spec.rb +137 -0
- data/spec/masamune/schema/event_spec.rb +75 -0
- data/spec/masamune/schema/fact_spec.rb +117 -0
- data/spec/masamune/schema/map_spec.rb +593 -0
- data/spec/masamune/schema/row_spec.rb +28 -0
- data/spec/masamune/schema/store_spec.rb +49 -0
- data/spec/masamune/schema/table_spec.rb +395 -0
- data/spec/masamune/string_format_spec.rb +60 -0
- data/spec/masamune/tasks/elastic_mapreduce_thor_spec.rb +57 -0
- data/spec/masamune/tasks/hive_thor_spec.rb +75 -0
- data/spec/masamune/tasks/postgres_thor_spec.rb +42 -0
- data/spec/masamune/tasks/shell_thor_spec.rb +51 -0
- data/spec/masamune/template_spec.rb +77 -0
- data/spec/masamune/thor_spec.rb +238 -0
- data/spec/masamune/transform/bulk_upsert.dimension_spec.rb +200 -0
- data/spec/masamune/transform/consolidate_dimension_spec.rb +62 -0
- data/spec/masamune/transform/deduplicate_dimension_spec.rb +84 -0
- data/spec/masamune/transform/define_event_view_spec.rb +84 -0
- data/spec/masamune/transform/define_schema_spec.rb +83 -0
- data/spec/masamune/transform/define_table.dimension_spec.rb +306 -0
- data/spec/masamune/transform/define_table.fact_spec.rb +291 -0
- data/spec/masamune/transform/define_table.table_spec.rb +525 -0
- data/spec/masamune/transform/insert_reference_values.dimension_spec.rb +111 -0
- data/spec/masamune/transform/insert_reference_values.fact_spec.rb +149 -0
- data/spec/masamune/transform/load_dimension_spec.rb +76 -0
- data/spec/masamune/transform/load_fact_spec.rb +89 -0
- data/spec/masamune/transform/relabel_dimension_spec.rb +102 -0
- data/spec/masamune/transform/rollup_fact_spec.rb +333 -0
- data/spec/masamune/transform/snapshot_dimension_spec.rb +103 -0
- data/spec/masamune/transform/stage_dimension_spec.rb +115 -0
- data/spec/masamune/transform/stage_fact_spec.rb +204 -0
- data/spec/masamune_spec.rb +32 -0
- data/spec/spec_helper.rb +41 -0
- data/spec/support/masamune/example_group.rb +36 -0
- data/spec/support/masamune/mock_command.rb +99 -0
- data/spec/support/masamune/mock_delegate.rb +51 -0
- data/spec/support/masamune/mock_filesystem.rb +96 -0
- data/spec/support/masamune/thor_mute.rb +35 -0
- data/spec/support/rspec/example/action_example_group.rb +34 -0
- data/spec/support/rspec/example/task_example_group.rb +80 -0
- data/spec/support/rspec/example/transform_example_group.rb +36 -0
- data/spec/support/shared_examples/postgres_common_examples.rb +53 -0
- metadata +462 -0
@@ -0,0 +1,204 @@
|
|
1
|
+
# The MIT License (MIT)
|
2
|
+
#
|
3
|
+
# Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
13
|
+
# all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
# THE SOFTWARE.
|
22
|
+
|
23
|
+
require 'spec_helper'
|
24
|
+
|
25
|
+
describe Masamune::Transform::StageFact do
|
26
|
+
before do
|
27
|
+
catalog.schema :postgres do
|
28
|
+
dimension 'cluster', type: :mini do
|
29
|
+
column 'id', type: :sequence, surrogate_key: true, auto: true
|
30
|
+
column 'name', type: :string
|
31
|
+
|
32
|
+
row name: 'current_database()', attributes: {default: true}
|
33
|
+
end
|
34
|
+
|
35
|
+
dimension 'date', type: :date do
|
36
|
+
column 'date_id', type: :integer, unique: true, index: true, natural_key: true
|
37
|
+
end
|
38
|
+
|
39
|
+
dimension 'user_agent', type: :mini do
|
40
|
+
column 'name', type: :string, unique: true, index: 'shared'
|
41
|
+
column 'version', type: :string, unique: true, index: 'shared', default: 'Unknown'
|
42
|
+
column 'mobile', type: :boolean, unique: true, index: 'shared', default: false
|
43
|
+
column 'description', type: :string, null: true, ignore: true
|
44
|
+
end
|
45
|
+
|
46
|
+
dimension 'feature', type: :mini do
|
47
|
+
column 'name', type: :string, unique: true, index: true
|
48
|
+
end
|
49
|
+
|
50
|
+
dimension 'tenant', type: :two do
|
51
|
+
column 'tenant_id', type: :integer, index: true, natural_key: true
|
52
|
+
end
|
53
|
+
|
54
|
+
dimension 'user', type: :two do
|
55
|
+
column 'tenant_id', type: :integer, index: true, natural_key: true
|
56
|
+
column 'user_id', type: :integer, index: true, natural_key: true
|
57
|
+
end
|
58
|
+
|
59
|
+
dimension 'group', type: :two do
|
60
|
+
column 'group_id', type: :integer, index: true, natural_key: true
|
61
|
+
column 'group_mode', type: :enum, sub_type: 'group_mode', values: %(missing public private), index: true, natural_key: true, default: 'missing'
|
62
|
+
row group_id: -1, group_mode: 'missing', attributes: {id: :missing}
|
63
|
+
end
|
64
|
+
|
65
|
+
fact 'visits', partition: 'y%Ym%m', grain: %w(hourly daily monthly) do
|
66
|
+
references :cluster
|
67
|
+
references :date
|
68
|
+
references :tenant
|
69
|
+
references :user
|
70
|
+
references :group, default: :missing
|
71
|
+
references :user_agent, insert: true
|
72
|
+
references :feature, insert: true
|
73
|
+
references :session, degenerate: true
|
74
|
+
measure 'total', type: :integer
|
75
|
+
end
|
76
|
+
|
77
|
+
file 'visits_hourly' do
|
78
|
+
column 'date.date_id', type: :integer
|
79
|
+
column 'tenant.tenant_id', type: :integer
|
80
|
+
column 'user.user_id', type: :integer
|
81
|
+
column 'group.group_id', type: :integer
|
82
|
+
column 'group.group_mode', type: :enum, sub_type: 'group_mode'
|
83
|
+
column 'user_agent.name', type: :string
|
84
|
+
column 'user_agent.version', type: :string
|
85
|
+
column 'user_agent.mobile', type: :boolean
|
86
|
+
column 'feature.name', type: :string
|
87
|
+
column 'session.id', type: :integer
|
88
|
+
column 'time_key', type: :integer
|
89
|
+
column 'total', type: :integer
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
93
|
+
|
94
|
+
let(:date) { DateTime.civil(2014,8) }
|
95
|
+
let(:target) { catalog.postgres.visits_hourly_fact }
|
96
|
+
let(:source) { catalog.postgres.visits_hourly_file.stage_table(suffix: 'file', table: target, inherit: false) }
|
97
|
+
|
98
|
+
context 'with postgres fact' do
|
99
|
+
subject(:result) { transform.stage_fact(source, target, date).to_s }
|
100
|
+
|
101
|
+
it 'should eq render stage_fact template' do
|
102
|
+
is_expected.to eq <<-EOS.strip_heredoc
|
103
|
+
BEGIN;
|
104
|
+
|
105
|
+
DROP TABLE IF EXISTS visits_hourly_fact_y2014m08_stage CASCADE;
|
106
|
+
CREATE TABLE IF NOT EXISTS visits_hourly_fact_y2014m08_stage (LIKE visits_hourly_fact INCLUDING ALL);
|
107
|
+
|
108
|
+
ALTER TABLE visits_hourly_fact_y2014m08_stage ADD CONSTRAINT visits_hourly_fact_y2014m08_stage_time_key_check CHECK (time_key >= 1406851200 AND time_key < 1409529600);
|
109
|
+
ALTER TABLE visits_hourly_fact_y2014m08_stage ADD CONSTRAINT visits_hourly_fact_y2014m08_stage_cluster_type_id_fkey FOREIGN KEY (cluster_type_id) REFERENCES cluster_type(id);
|
110
|
+
ALTER TABLE visits_hourly_fact_y2014m08_stage ADD CONSTRAINT visits_hourly_fact_y2014m08_stage_date_dimension_id_fkey FOREIGN KEY (date_dimension_id) REFERENCES date_dimension(id);
|
111
|
+
ALTER TABLE visits_hourly_fact_y2014m08_stage ADD CONSTRAINT visits_hourly_fact_y2014m08_stage_tenant_dimension_id_fkey FOREIGN KEY (tenant_dimension_id) REFERENCES tenant_dimension(id);
|
112
|
+
ALTER TABLE visits_hourly_fact_y2014m08_stage ADD CONSTRAINT visits_hourly_fact_y2014m08_stage_user_dimension_id_fkey FOREIGN KEY (user_dimension_id) REFERENCES user_dimension(id);
|
113
|
+
ALTER TABLE visits_hourly_fact_y2014m08_stage ADD CONSTRAINT visits_hourly_fact_y2014m08_stage_group_dimension_id_fkey FOREIGN KEY (group_dimension_id) REFERENCES group_dimension(id);
|
114
|
+
ALTER TABLE visits_hourly_fact_y2014m08_stage ADD CONSTRAINT visits_hourly_fact_y2014m08_stage_user_agent_type_id_fkey FOREIGN KEY (user_agent_type_id) REFERENCES user_agent_type(id);
|
115
|
+
ALTER TABLE visits_hourly_fact_y2014m08_stage ADD CONSTRAINT visits_hourly_fact_y2014m08_stage_feature_type_id_fkey FOREIGN KEY (feature_type_id) REFERENCES feature_type(id);
|
116
|
+
|
117
|
+
INSERT INTO
|
118
|
+
visits_hourly_fact_y2014m08_stage (date_dimension_id, tenant_dimension_id, user_dimension_id, group_dimension_id, user_agent_type_id, feature_type_id, session_type_id, total, time_key)
|
119
|
+
SELECT
|
120
|
+
date_dimension.id,
|
121
|
+
tenant_dimension.id,
|
122
|
+
user_dimension.id,
|
123
|
+
group_dimension.id,
|
124
|
+
user_agent_type.id,
|
125
|
+
feature_type.id,
|
126
|
+
visits_hourly_file_fact_stage.session_type_id,
|
127
|
+
visits_hourly_file_fact_stage.total,
|
128
|
+
visits_hourly_file_fact_stage.time_key
|
129
|
+
FROM
|
130
|
+
visits_hourly_file_fact_stage
|
131
|
+
JOIN
|
132
|
+
date_dimension
|
133
|
+
ON
|
134
|
+
date_dimension.date_id = visits_hourly_file_fact_stage.date_dimension_date_id
|
135
|
+
JOIN
|
136
|
+
user_dimension
|
137
|
+
ON
|
138
|
+
user_dimension.user_id = visits_hourly_file_fact_stage.user_dimension_user_id AND
|
139
|
+
((TO_TIMESTAMP(visits_hourly_file_fact_stage.time_key) BETWEEN user_dimension.start_at AND COALESCE(user_dimension.end_at, 'INFINITY')) OR (TO_TIMESTAMP(visits_hourly_file_fact_stage.time_key) < user_dimension.start_at AND user_dimension.version = 1))
|
140
|
+
JOIN
|
141
|
+
tenant_dimension
|
142
|
+
ON
|
143
|
+
tenant_dimension.tenant_id = COALESCE(visits_hourly_file_fact_stage.tenant_dimension_tenant_id, user_dimension.tenant_id) AND
|
144
|
+
((TO_TIMESTAMP(visits_hourly_file_fact_stage.time_key) BETWEEN tenant_dimension.start_at AND COALESCE(tenant_dimension.end_at, 'INFINITY')) OR (TO_TIMESTAMP(visits_hourly_file_fact_stage.time_key) < tenant_dimension.start_at AND tenant_dimension.version = 1))
|
145
|
+
JOIN
|
146
|
+
group_dimension
|
147
|
+
ON
|
148
|
+
group_dimension.group_id = COALESCE(visits_hourly_file_fact_stage.group_dimension_group_id, missing_group_id()) AND
|
149
|
+
group_dimension.group_mode = COALESCE(visits_hourly_file_fact_stage.group_dimension_group_mode, missing_group_mode()) AND
|
150
|
+
((TO_TIMESTAMP(visits_hourly_file_fact_stage.time_key) BETWEEN group_dimension.start_at AND COALESCE(group_dimension.end_at, 'INFINITY')) OR (TO_TIMESTAMP(visits_hourly_file_fact_stage.time_key) < group_dimension.start_at AND group_dimension.version = 1))
|
151
|
+
JOIN
|
152
|
+
user_agent_type
|
153
|
+
ON
|
154
|
+
user_agent_type.name = visits_hourly_file_fact_stage.user_agent_type_name AND
|
155
|
+
user_agent_type.version = COALESCE(visits_hourly_file_fact_stage.user_agent_type_version, 'Unknown') AND
|
156
|
+
user_agent_type.mobile = COALESCE(visits_hourly_file_fact_stage.user_agent_type_mobile, FALSE)
|
157
|
+
JOIN
|
158
|
+
feature_type
|
159
|
+
ON
|
160
|
+
feature_type.name = visits_hourly_file_fact_stage.feature_type_name
|
161
|
+
;
|
162
|
+
|
163
|
+
CREATE INDEX visits_hourly_fact_y2014m08_stage_cluster_type_id_index ON visits_hourly_fact_y2014m08_stage (cluster_type_id);
|
164
|
+
CREATE INDEX visits_hourly_fact_y2014m08_stage_date_dimension_id_index ON visits_hourly_fact_y2014m08_stage (date_dimension_id);
|
165
|
+
CREATE INDEX visits_hourly_fact_y2014m08_stage_tenant_dimension_id_index ON visits_hourly_fact_y2014m08_stage (tenant_dimension_id);
|
166
|
+
CREATE INDEX visits_hourly_fact_y2014m08_stage_user_dimension_id_index ON visits_hourly_fact_y2014m08_stage (user_dimension_id);
|
167
|
+
CREATE INDEX visits_hourly_fact_y2014m08_stage_group_dimension_id_index ON visits_hourly_fact_y2014m08_stage (group_dimension_id);
|
168
|
+
CREATE INDEX visits_hourly_fact_y2014m08_stage_user_agent_type_id_index ON visits_hourly_fact_y2014m08_stage (user_agent_type_id);
|
169
|
+
CREATE INDEX visits_hourly_fact_y2014m08_stage_feature_type_id_index ON visits_hourly_fact_y2014m08_stage (feature_type_id);
|
170
|
+
CREATE INDEX visits_hourly_fact_y2014m08_stage_session_type_id_index ON visits_hourly_fact_y2014m08_stage (session_type_id);
|
171
|
+
CREATE INDEX visits_hourly_fact_y2014m08_stage_time_key_index ON visits_hourly_fact_y2014m08_stage (time_key);
|
172
|
+
|
173
|
+
COMMIT;
|
174
|
+
|
175
|
+
BEGIN;
|
176
|
+
|
177
|
+
DROP TABLE IF EXISTS visits_hourly_fact_y2014m08;
|
178
|
+
ALTER TABLE visits_hourly_fact_y2014m08_stage RENAME TO visits_hourly_fact_y2014m08;
|
179
|
+
|
180
|
+
ALTER TABLE visits_hourly_fact_y2014m08 INHERIT visits_hourly_fact;
|
181
|
+
ALTER TABLE visits_hourly_fact_y2014m08 ADD CONSTRAINT visits_hourly_fact_y2014m08_time_key_check CHECK (time_key >= 1406851200 AND time_key < 1409529600) NOT VALID;
|
182
|
+
ALTER TABLE visits_hourly_fact_y2014m08 ADD CONSTRAINT visits_hourly_fact_y2014m08_cluster_type_id_fkey FOREIGN KEY (cluster_type_id) REFERENCES cluster_type(id) NOT VALID;
|
183
|
+
ALTER TABLE visits_hourly_fact_y2014m08 ADD CONSTRAINT visits_hourly_fact_y2014m08_date_dimension_id_fkey FOREIGN KEY (date_dimension_id) REFERENCES date_dimension(id) NOT VALID;
|
184
|
+
ALTER TABLE visits_hourly_fact_y2014m08 ADD CONSTRAINT visits_hourly_fact_y2014m08_tenant_dimension_id_fkey FOREIGN KEY (tenant_dimension_id) REFERENCES tenant_dimension(id) NOT VALID;
|
185
|
+
ALTER TABLE visits_hourly_fact_y2014m08 ADD CONSTRAINT visits_hourly_fact_y2014m08_user_dimension_id_fkey FOREIGN KEY (user_dimension_id) REFERENCES user_dimension(id) NOT VALID;
|
186
|
+
ALTER TABLE visits_hourly_fact_y2014m08 ADD CONSTRAINT visits_hourly_fact_y2014m08_group_dimension_id_fkey FOREIGN KEY (group_dimension_id) REFERENCES group_dimension(id) NOT VALID;
|
187
|
+
ALTER TABLE visits_hourly_fact_y2014m08 ADD CONSTRAINT visits_hourly_fact_y2014m08_user_agent_type_id_fkey FOREIGN KEY (user_agent_type_id) REFERENCES user_agent_type(id) NOT VALID;
|
188
|
+
ALTER TABLE visits_hourly_fact_y2014m08 ADD CONSTRAINT visits_hourly_fact_y2014m08_feature_type_id_fkey FOREIGN KEY (feature_type_id) REFERENCES feature_type(id) NOT VALID;
|
189
|
+
|
190
|
+
ALTER INDEX visits_hourly_fact_y2014m08_stage_cluster_type_id_index RENAME TO visits_hourly_fact_y2014m08_cluster_type_id_index;
|
191
|
+
ALTER INDEX visits_hourly_fact_y2014m08_stage_date_dimension_id_index RENAME TO visits_hourly_fact_y2014m08_date_dimension_id_index;
|
192
|
+
ALTER INDEX visits_hourly_fact_y2014m08_stage_tenant_dimension_id_index RENAME TO visits_hourly_fact_y2014m08_tenant_dimension_id_index;
|
193
|
+
ALTER INDEX visits_hourly_fact_y2014m08_stage_user_dimension_id_index RENAME TO visits_hourly_fact_y2014m08_user_dimension_id_index;
|
194
|
+
ALTER INDEX visits_hourly_fact_y2014m08_stage_group_dimension_id_index RENAME TO visits_hourly_fact_y2014m08_group_dimension_id_index;
|
195
|
+
ALTER INDEX visits_hourly_fact_y2014m08_stage_user_agent_type_id_index RENAME TO visits_hourly_fact_y2014m08_user_agent_type_id_index;
|
196
|
+
ALTER INDEX visits_hourly_fact_y2014m08_stage_feature_type_id_index RENAME TO visits_hourly_fact_y2014m08_feature_type_id_index;
|
197
|
+
ALTER INDEX visits_hourly_fact_y2014m08_stage_session_type_id_index RENAME TO visits_hourly_fact_y2014m08_session_type_id_index;
|
198
|
+
ALTER INDEX visits_hourly_fact_y2014m08_stage_time_key_index RENAME TO visits_hourly_fact_y2014m08_time_key_index;
|
199
|
+
|
200
|
+
COMMIT;
|
201
|
+
EOS
|
202
|
+
end
|
203
|
+
end
|
204
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
# The MIT License (MIT)
|
2
|
+
#
|
3
|
+
# Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
13
|
+
# all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
# THE SOFTWARE.
|
22
|
+
|
23
|
+
require 'spec_helper'
|
24
|
+
|
25
|
+
describe Masamune do
|
26
|
+
it { is_expected.to be_a(Module) }
|
27
|
+
|
28
|
+
describe '#environment' do
|
29
|
+
subject { described_class.environment }
|
30
|
+
it { is_expected.to be_a(Masamune::Environment) }
|
31
|
+
end
|
32
|
+
end
|
data/spec/spec_helper.rb
ADDED
@@ -0,0 +1,41 @@
|
|
1
|
+
# The MIT License (MIT)
|
2
|
+
#
|
3
|
+
# Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
13
|
+
# all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
# THE SOFTWARE.
|
22
|
+
|
23
|
+
require 'bundler/setup'
|
24
|
+
Bundler.require(:default, :development, :test)
|
25
|
+
|
26
|
+
require 'tempfile'
|
27
|
+
require 'tmpdir'
|
28
|
+
|
29
|
+
Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
|
30
|
+
|
31
|
+
require 'active_support/core_ext/string/strip'
|
32
|
+
|
33
|
+
Masamune::ExampleGroup.configure do |config|
|
34
|
+
config.quiet = ENV['MASAMUNE_DEBUG'] ? false : true
|
35
|
+
config.debug = ENV['MASAMUNE_DEBUG'] ? true : false
|
36
|
+
config.retries = 0
|
37
|
+
end
|
38
|
+
|
39
|
+
RSpec.configure do |config|
|
40
|
+
config.mock_with :rspec
|
41
|
+
end
|
@@ -0,0 +1,36 @@
|
|
1
|
+
# The MIT License (MIT)
|
2
|
+
#
|
3
|
+
# Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
13
|
+
# all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
# THE SOFTWARE.
|
22
|
+
|
23
|
+
require 'masamune/has_environment'
|
24
|
+
|
25
|
+
# Separate environment for test harness itself
|
26
|
+
module Masamune::ExampleGroup
|
27
|
+
include Masamune::HasEnvironment
|
28
|
+
extend self
|
29
|
+
|
30
|
+
def self.included(base)
|
31
|
+
base.before(:all) do
|
32
|
+
self.filesystem.environment = self.environment = Masamune::ExampleGroup.environment
|
33
|
+
Thor.send(:include, Masamune::ThorMute)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
@@ -0,0 +1,99 @@
|
|
1
|
+
# The MIT License (MIT)
|
2
|
+
#
|
3
|
+
# Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
13
|
+
# all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
# THE SOFTWARE.
|
22
|
+
|
23
|
+
require 'delegate'
|
24
|
+
require 'active_support/concern'
|
25
|
+
|
26
|
+
module Masamune::MockCommand
|
27
|
+
extend ActiveSupport::Concern
|
28
|
+
|
29
|
+
class CommandMatcher < SimpleDelegator
|
30
|
+
def initialize(delegate)
|
31
|
+
super delegate
|
32
|
+
@delegate = delegate
|
33
|
+
end
|
34
|
+
|
35
|
+
class << self
|
36
|
+
def add_pattern(pattern, value, io)
|
37
|
+
@patterns ||= {}
|
38
|
+
@patterns[pattern] = [value, io]
|
39
|
+
end
|
40
|
+
|
41
|
+
def patterns
|
42
|
+
@patterns ||= {}
|
43
|
+
@patterns
|
44
|
+
end
|
45
|
+
|
46
|
+
def reset!
|
47
|
+
@patterns = {}
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
def around_execute(&block)
|
52
|
+
self.class.patterns.each do |pattern, (value, io)|
|
53
|
+
if @delegate.command_args.join(' ') =~ pattern
|
54
|
+
while line = io.gets
|
55
|
+
line_no ||= 0
|
56
|
+
@delegate.handle_stdout(line.chomp, line_no) if @delegate.respond_to?(:handle_stdout)
|
57
|
+
line_no += 1
|
58
|
+
end
|
59
|
+
return value.respond_to?(:call) ? value.call : value
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
if @delegate.respond_to?(:around_execute)
|
64
|
+
@delegate.around_execute(&block)
|
65
|
+
else
|
66
|
+
block.call
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
|
71
|
+
included do |base|
|
72
|
+
base.before do
|
73
|
+
new_method = Masamune::Commands::Shell.method(:new)
|
74
|
+
allow(Masamune::Commands::Shell).to receive(:new) do |command, options|
|
75
|
+
new_method.call(CommandMatcher.new(command), options || {})
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
base.after do
|
80
|
+
CommandMatcher.reset!
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def mock_success
|
85
|
+
OpenStruct.new(:success? => true)
|
86
|
+
end
|
87
|
+
|
88
|
+
def mock_failure
|
89
|
+
OpenStruct.new(:success? => false)
|
90
|
+
end
|
91
|
+
|
92
|
+
def mock_command(pattern, value = nil, io = StringIO.new, &block)
|
93
|
+
CommandMatcher.add_pattern(pattern, block_given? ? block.to_proc : value, io, &block)
|
94
|
+
end
|
95
|
+
end
|
96
|
+
|
97
|
+
RSpec.configure do |config|
|
98
|
+
config.include Masamune::MockCommand
|
99
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
# The MIT License (MIT)
|
2
|
+
#
|
3
|
+
# Copyright (c) 2014-2015, VMware, Inc. All Rights Reserved.
|
4
|
+
#
|
5
|
+
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
# of this software and associated documentation files (the "Software"), to deal
|
7
|
+
# in the Software without restriction, including without limitation the rights
|
8
|
+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
# copies of the Software, and to permit persons to whom the Software is
|
10
|
+
# furnished to do so, subject to the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be included in
|
13
|
+
# all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
# THE SOFTWARE.
|
22
|
+
|
23
|
+
class Masamune::MockDelegate
|
24
|
+
include Masamune::HasEnvironment
|
25
|
+
|
26
|
+
attr_accessor :command, :stdin, :stdout, :stderr, :status
|
27
|
+
|
28
|
+
def initialize(command, input = nil)
|
29
|
+
self.command = command
|
30
|
+
self.stdin = StringIO.new(input) if input
|
31
|
+
self.stdout = []
|
32
|
+
self.stderr = []
|
33
|
+
self.status = 0
|
34
|
+
end
|
35
|
+
|
36
|
+
def command_args
|
37
|
+
[command]
|
38
|
+
end
|
39
|
+
|
40
|
+
def handle_stdout(line, line_no)
|
41
|
+
self.stdout[line_no] = line
|
42
|
+
end
|
43
|
+
|
44
|
+
def handle_stderr(line, line_no)
|
45
|
+
self.stderr[line_no] = line
|
46
|
+
end
|
47
|
+
|
48
|
+
def handle_failure(code)
|
49
|
+
self.status = code
|
50
|
+
end
|
51
|
+
end
|