masamune 0.17.12 → 0.17.13
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Rakefile +3 -3
- data/bin/masamune-aws-emr +1 -1
- data/bin/masamune-dump +1 -1
- data/bin/masamune-hive +1 -1
- data/bin/masamune-psql +1 -1
- data/bin/masamune-shell +1 -1
- data/lib/masamune.rb +1 -1
- data/lib/masamune/actions/aws_emr.rb +2 -2
- data/lib/masamune/actions/data_flow.rb +10 -10
- data/lib/masamune/actions/date_parse.rb +4 -6
- data/lib/masamune/actions/execute.rb +3 -3
- data/lib/masamune/actions/filesystem.rb +1 -1
- data/lib/masamune/actions/hadoop_filesystem.rb +2 -2
- data/lib/masamune/actions/hive.rb +5 -5
- data/lib/masamune/actions/invoke_parallel.rb +5 -5
- data/lib/masamune/actions/postgres.rb +6 -6
- data/lib/masamune/actions/postgres_admin.rb +1 -1
- data/lib/masamune/actions/s3cmd.rb +3 -3
- data/lib/masamune/actions/transform.rb +4 -4
- data/lib/masamune/after_initialize_callbacks.rb +4 -4
- data/lib/masamune/cached_filesystem.rb +7 -11
- data/lib/masamune/commands/aws_emr.rb +12 -12
- data/lib/masamune/commands/hadoop_filesystem.rb +8 -8
- data/lib/masamune/commands/hadoop_streaming.rb +16 -16
- data/lib/masamune/commands/hive.rb +31 -35
- data/lib/masamune/commands/postgres.rb +30 -32
- data/lib/masamune/commands/postgres_admin.rb +17 -16
- data/lib/masamune/commands/postgres_common.rb +1 -1
- data/lib/masamune/commands/retry_with_backoff.rb +19 -19
- data/lib/masamune/commands/s3cmd.rb +7 -7
- data/lib/masamune/commands/shell.rb +25 -38
- data/lib/masamune/configuration.rb +8 -9
- data/lib/masamune/data_plan/builder.rb +5 -5
- data/lib/masamune/data_plan/elem.rb +7 -7
- data/lib/masamune/data_plan/engine.rb +12 -15
- data/lib/masamune/data_plan/rule.rb +14 -12
- data/lib/masamune/data_plan/set.rb +13 -13
- data/lib/masamune/environment.rb +3 -3
- data/lib/masamune/filesystem.rb +29 -28
- data/lib/masamune/helpers/postgres.rb +8 -5
- data/lib/masamune/io.rb +2 -2
- data/lib/masamune/last_element.rb +3 -3
- data/lib/masamune/method_logger.rb +1 -1
- data/lib/masamune/schema/catalog.rb +9 -12
- data/lib/masamune/schema/column.rb +12 -10
- data/lib/masamune/schema/dimension.rb +5 -4
- data/lib/masamune/schema/fact.rb +2 -2
- data/lib/masamune/schema/map.rb +9 -13
- data/lib/masamune/schema/row.rb +3 -2
- data/lib/masamune/schema/store.rb +13 -14
- data/lib/masamune/schema/table.rb +24 -28
- data/lib/masamune/schema/table_reference.rb +7 -7
- data/lib/masamune/spec_helper.rb +1 -1
- data/lib/masamune/string_format.rb +1 -1
- data/lib/masamune/tasks/aws_emr_thor.rb +9 -9
- data/lib/masamune/tasks/dump_thor.rb +5 -5
- data/lib/masamune/tasks/hive_thor.rb +13 -13
- data/lib/masamune/tasks/postgres_thor.rb +8 -8
- data/lib/masamune/tasks/shell_thor.rb +3 -3
- data/lib/masamune/template.rb +4 -4
- data/lib/masamune/thor.rb +26 -25
- data/lib/masamune/transform/common/denormalize_table.rb +6 -6
- data/lib/masamune/transform/define_table.rb +1 -1
- data/lib/masamune/transform/hive/define_table.rb +0 -2
- data/lib/masamune/transform/insert_reference_values.rb +1 -1
- data/lib/masamune/transform/operator.rb +3 -2
- data/lib/masamune/transform/postgres/bulk_upsert.rb +0 -3
- data/lib/masamune/transform/postgres/deduplicate_dimension.rb +4 -6
- data/lib/masamune/transform/postgres/define_table.rb +2 -2
- data/lib/masamune/transform/postgres/insert_reference_values.rb +1 -4
- data/lib/masamune/transform/postgres/relabel_dimension.rb +1 -3
- data/lib/masamune/transform/postgres/rollup_fact.rb +3 -5
- data/lib/masamune/transform/postgres/snapshot_dimension.rb +1 -4
- data/lib/masamune/transform/postgres/stage_dimension.rb +8 -10
- data/lib/masamune/transform/postgres/stage_fact.rb +15 -14
- data/lib/masamune/version.rb +1 -1
- data/spec/masamune/actions/aws_emr_spec.rb +8 -8
- data/spec/masamune/actions/execute_spec.rb +6 -6
- data/spec/masamune/actions/hive_spec.rb +20 -15
- data/spec/masamune/actions/postgres_spec.rb +17 -10
- data/spec/masamune/after_initialization_callbacks_spec.rb +13 -7
- data/spec/masamune/commands/aws_emr_spec.rb +10 -10
- data/spec/masamune/commands/hadoop_filesystem_spec.rb +3 -3
- data/spec/masamune/commands/hadoop_streaming_spec.rb +4 -4
- data/spec/masamune/commands/hive_spec.rb +11 -11
- data/spec/masamune/commands/postgres_admin_spec.rb +9 -9
- data/spec/masamune/commands/postgres_spec.rb +15 -15
- data/spec/masamune/commands/retry_with_backoff_spec.rb +10 -7
- data/spec/masamune/commands/s3cmd_spec.rb +3 -3
- data/spec/masamune/commands/shell_spec.rb +16 -16
- data/spec/masamune/data_plan/builder_spec.rb +22 -19
- data/spec/masamune/data_plan/elem_spec.rb +20 -20
- data/spec/masamune/data_plan/engine_spec.rb +41 -33
- data/spec/masamune/data_plan/rule_spec.rb +19 -19
- data/spec/masamune/data_plan/set_spec.rb +8 -6
- data/spec/masamune/environment_spec.rb +5 -5
- data/spec/masamune/filesystem_spec.rb +87 -87
- data/spec/masamune/helpers/postgres_spec.rb +1 -1
- data/spec/masamune/rspec/job_fixture_spec.rb +0 -1
- data/spec/masamune/schema/catalog_spec.rb +16 -16
- data/spec/masamune/schema/column_spec.rb +19 -19
- data/spec/masamune/schema/dimension_spec.rb +2 -2
- data/spec/masamune/schema/fact_spec.rb +3 -3
- data/spec/masamune/schema/map_spec.rb +41 -41
- data/spec/masamune/schema/table_spec.rb +16 -16
- data/spec/masamune/string_format_spec.rb +7 -7
- data/spec/masamune/tasks/dump_thor_spec.rb +6 -6
- data/spec/masamune/tasks/hive_thor_spec.rb +2 -2
- data/spec/masamune/template_spec.rb +3 -2
- data/spec/masamune/thor_spec.rb +8 -8
- data/spec/masamune/transform/bulk_upsert.dimension_spec.rb +3 -3
- data/spec/masamune/transform/define_table.dimension_spec.rb +3 -3
- data/spec/masamune/transform/define_table.fact_spec.rb +5 -5
- data/spec/masamune/transform/define_table.table_spec.rb +6 -6
- data/spec/masamune/transform/denormalize_table_spec.rb +15 -15
- data/spec/masamune/transform/insert_reference_values.dimension_spec.rb +1 -1
- data/spec/masamune/transform/load_dimension_spec.rb +1 -1
- data/spec/masamune/transform/load_fact_spec.rb +1 -1
- data/spec/masamune/transform/rollup_fact_spec.rb +4 -4
- data/spec/masamune/transform/stage_dimension_spec.rb +3 -3
- data/spec/masamune/transform/stage_fact_spec.rb +4 -4
- data/spec/spec_helper.rb +1 -1
- data/spec/support/masamune/example_group.rb +2 -2
- data/spec/support/masamune/job_example_group.rb +3 -3
- data/spec/support/masamune/job_fixture.rb +4 -4
- data/spec/support/masamune/mock_command.rb +10 -10
- data/spec/support/masamune/mock_delegate.rb +2 -2
- data/spec/support/masamune/mock_filesystem.rb +3 -3
- data/spec/support/masamune/shared_example_group.rb +14 -16
- data/spec/support/masamune/step_example_group.rb +4 -4
- data/spec/support/masamune/step_fixture.rb +1 -1
- data/spec/support/masamune/task_example_group.rb +1 -1
- data/spec/support/masamune/thor_mute.rb +3 -2
- data/spec/support/rspec/example/action_example_group.rb +1 -1
- data/spec/support/rspec/example/task_example_group.rb +7 -4
- data/spec/support/rspec/example/transform_example_group.rb +1 -1
- data/spec/support/shared_examples/postgres_common_examples.rb +2 -2
- metadata +16 -2
@@ -69,7 +69,7 @@ describe Masamune::Transform::LoadFact do
|
|
69
69
|
end
|
70
70
|
|
71
71
|
let(:files) { (1..3).map { |i| double(path: "output_#{i}.csv") } }
|
72
|
-
let(:date) { DateTime.civil(2014,8) }
|
72
|
+
let(:date) { DateTime.civil(2014, 8) }
|
73
73
|
let(:target) { catalog.postgres.visits_fact }
|
74
74
|
let(:source) { catalog.postgres.visits_file }
|
75
75
|
let(:source_table) { source.stage_table(suffix: 'file', table: target, inherit: false) }
|
@@ -29,7 +29,7 @@ describe Masamune::Transform::RollupFact do
|
|
29
29
|
column 'id', type: :sequence, surrogate_key: true, auto: true
|
30
30
|
column 'name', type: :string
|
31
31
|
|
32
|
-
row name: 'current_database()', attributes: {default: true}
|
32
|
+
row name: 'current_database()', attributes: { default: true }
|
33
33
|
end
|
34
34
|
|
35
35
|
dimension 'date', type: :date do
|
@@ -82,7 +82,7 @@ describe Masamune::Transform::RollupFact do
|
|
82
82
|
end
|
83
83
|
|
84
84
|
context 'with postgres transaction fact' do
|
85
|
-
let(:date) { DateTime.civil(2014,8) }
|
85
|
+
let(:date) { DateTime.civil(2014, 8) }
|
86
86
|
let(:source) { catalog.postgres.visits_transaction_fact }
|
87
87
|
let(:target) { catalog.postgres.visits_hourly_fact }
|
88
88
|
|
@@ -180,7 +180,7 @@ describe Masamune::Transform::RollupFact do
|
|
180
180
|
end
|
181
181
|
|
182
182
|
context 'with postgres hourly fact' do
|
183
|
-
let(:date) { DateTime.civil(2014,8) }
|
183
|
+
let(:date) { DateTime.civil(2014, 8) }
|
184
184
|
let(:source) { catalog.postgres.visits_hourly_fact }
|
185
185
|
let(:target) { catalog.postgres.visits_daily_fact }
|
186
186
|
|
@@ -278,7 +278,7 @@ describe Masamune::Transform::RollupFact do
|
|
278
278
|
end
|
279
279
|
|
280
280
|
context 'with postgres daily fact' do
|
281
|
-
let(:date) { DateTime.civil(2014,8) }
|
281
|
+
let(:date) { DateTime.civil(2014, 8) }
|
282
282
|
let(:source) { catalog.postgres.visits_daily_fact }
|
283
283
|
let(:target) { catalog.postgres.visits_monthly_fact }
|
284
284
|
|
@@ -26,14 +26,14 @@ describe Masamune::Transform::StageDimension do
|
|
26
26
|
dimension 'cluster', type: :mini do
|
27
27
|
column 'id', type: :integer, surrogate_key: true, auto: true
|
28
28
|
column 'name', type: :string, unique: true
|
29
|
-
row name: 'default', attributes: {default: true}
|
29
|
+
row name: 'default', attributes: { default: true }
|
30
30
|
end
|
31
31
|
|
32
32
|
dimension 'user_account_state', type: :mini do
|
33
33
|
column 'name', type: :string, unique: true
|
34
34
|
column 'description', type: :string
|
35
35
|
row name: 'registered', description: 'Registered'
|
36
|
-
row name: 'active', description: 'Active', attributes: {default: true}
|
36
|
+
row name: 'active', description: 'Active', attributes: { default: true }
|
37
37
|
row name: 'inactive', description: 'Inactive'
|
38
38
|
end
|
39
39
|
|
@@ -41,7 +41,7 @@ describe Masamune::Transform::StageDimension do
|
|
41
41
|
references :cluster
|
42
42
|
column 'tenant_id', type: :integer, unique: true, natural_key: true
|
43
43
|
column 'department_id', type: :integer, unique: true, natural_key: true
|
44
|
-
row tenant_id: -1, department_id: -1, attributes: {default: true}
|
44
|
+
row tenant_id: -1, department_id: -1, attributes: { default: true }
|
45
45
|
end
|
46
46
|
|
47
47
|
dimension 'user', type: :four do
|
@@ -29,7 +29,7 @@ describe Masamune::Transform::StageFact do
|
|
29
29
|
column 'id', type: :sequence, surrogate_key: true, auto: true
|
30
30
|
column 'name', type: :string
|
31
31
|
|
32
|
-
row name: 'current_database()', attributes: {default: true}
|
32
|
+
row name: 'current_database()', attributes: { default: true }
|
33
33
|
end
|
34
34
|
|
35
35
|
dimension 'date', type: :date do
|
@@ -68,8 +68,8 @@ describe Masamune::Transform::StageFact do
|
|
68
68
|
column 'tenant_id', type: :integer, natural_key: true
|
69
69
|
column 'group_id', type: :integer, natural_key: true
|
70
70
|
column 'group_mode', type: :enum, sub_type: 'group_mode', values: %(missing unknown public private), index: true, natural_key: true, default: 'missing'
|
71
|
-
row group_id: -1, group_mode: 'missing', attributes: {id: :missing}
|
72
|
-
row group_id: -2, group_mode: 'unknown', attributes: {id: :unknown}
|
71
|
+
row group_id: -1, group_mode: 'missing', attributes: { id: :missing }
|
72
|
+
row group_id: -2, group_mode: 'unknown', attributes: { id: :unknown }
|
73
73
|
end
|
74
74
|
|
75
75
|
fact 'visits', partition: 'y%Ym%m', grain: %w(hourly daily monthly) do
|
@@ -104,7 +104,7 @@ describe Masamune::Transform::StageFact do
|
|
104
104
|
end
|
105
105
|
end
|
106
106
|
|
107
|
-
let(:date) { DateTime.civil(2014,8) }
|
107
|
+
let(:date) { DateTime.civil(2014, 8) }
|
108
108
|
let(:target) { catalog.postgres.visits_hourly_fact }
|
109
109
|
let(:source) { catalog.postgres.visits_hourly_file.stage_table(suffix: 'file', table: target, inherit: false) }
|
110
110
|
|
data/spec/spec_helper.rb
CHANGED
@@ -26,7 +26,7 @@ Bundler.require(:default, :development, :test)
|
|
26
26
|
require 'tempfile'
|
27
27
|
require 'tmpdir'
|
28
28
|
|
29
|
-
Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
|
29
|
+
Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each { |f| require f }
|
30
30
|
|
31
31
|
require 'active_support/core_ext/string/strip'
|
32
32
|
|
@@ -25,11 +25,11 @@ require 'masamune/has_environment'
|
|
25
25
|
# Separate environment for test harness itself
|
26
26
|
module Masamune::ExampleGroup
|
27
27
|
include Masamune::HasEnvironment
|
28
|
-
extend self
|
28
|
+
extend self # rubocop:disable Style/ModuleFunction
|
29
29
|
|
30
30
|
def self.included(base)
|
31
31
|
base.before(:all) do
|
32
|
-
|
32
|
+
filesystem.environment = self.environment = Masamune::ExampleGroup.environment
|
33
33
|
Thor.send(:include, Masamune::ThorMute)
|
34
34
|
end
|
35
35
|
end
|
@@ -38,7 +38,7 @@ module Masamune::JobExampleGroup
|
|
38
38
|
it "should match #{fixture_file}" do
|
39
39
|
aggregate_failures 'generates expected output' do
|
40
40
|
gather_example_output(fixture) do |actual_data, expect_file, expect_data|
|
41
|
-
expect(File.
|
41
|
+
expect(File.exist?(expect_file)).to eq(true)
|
42
42
|
expect(actual_data).to eq(expect_data)
|
43
43
|
end
|
44
44
|
end
|
@@ -57,6 +57,6 @@ module Masamune::JobExampleGroup
|
|
57
57
|
end
|
58
58
|
|
59
59
|
RSpec.configure do |config|
|
60
|
-
config.include Masamune::JobExampleGroup, :
|
61
|
-
config.include Masamune::JobExampleGroup, :
|
60
|
+
config.include Masamune::JobExampleGroup, type: :job, file_path: %r{.*/spec/.*job_spec\.rb}
|
61
|
+
config.include Masamune::JobExampleGroup, type: :task, file_path: %r{.*/spec/.*task_spec\.rb}
|
62
62
|
end
|
@@ -43,7 +43,7 @@ module Masamune
|
|
43
43
|
class << self
|
44
44
|
def load(options = {}, context = binding)
|
45
45
|
file = file_name(options)
|
46
|
-
raise ArgumentError, "Fixture '#{file}' does not exist" unless File.
|
46
|
+
raise ArgumentError, "Fixture '#{file}' does not exist" unless File.exist?(file)
|
47
47
|
YAML.load(ERB.new(File.read(file)).result(context)).tap do |data|
|
48
48
|
return new(options.merge(data: data, context: context))
|
49
49
|
end
|
@@ -77,7 +77,7 @@ module Masamune
|
|
77
77
|
@data['inputs'].each do |input|
|
78
78
|
file.puts '-'.indent(INDENT)
|
79
79
|
serialize(input) do |elem|
|
80
|
-
file.puts elem.indent(INDENT*2)
|
80
|
+
file.puts elem.indent(INDENT * 2)
|
81
81
|
end
|
82
82
|
end
|
83
83
|
file.puts
|
@@ -85,7 +85,7 @@ module Masamune
|
|
85
85
|
@data['outputs'].each do |output|
|
86
86
|
file.puts '-'.indent(INDENT)
|
87
87
|
serialize(output) do |elem|
|
88
|
-
file.puts elem.indent(INDENT*2)
|
88
|
+
file.puts elem.indent(INDENT * 2)
|
89
89
|
end
|
90
90
|
end
|
91
91
|
end
|
@@ -97,7 +97,7 @@ module Masamune
|
|
97
97
|
@data['inputs'].map do |input|
|
98
98
|
if input['reference']
|
99
99
|
raise ArgumentError, "reference in #{file_name} requires fixture" unless input['reference']['fixture'] || input['reference']['file']
|
100
|
-
reference = self.class.load({path: input['reference']['path'] || path, name: input['reference']['fixture'], file: input['reference']['file'], type: @type}, @context)
|
100
|
+
reference = self.class.load({ path: input['reference']['path'] || path, name: input['reference']['fixture'], file: input['reference']['file'], type: @type }, @context)
|
101
101
|
section = input['reference']['section'] || 'outputs'
|
102
102
|
reference.send(section) if reference.respond_to?(section)
|
103
103
|
else
|
@@ -50,20 +50,20 @@ module Masamune::MockCommand
|
|
50
50
|
|
51
51
|
def around_execute(&block)
|
52
52
|
self.class.patterns.each do |pattern, (value, io)|
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
return value.respond_to?(:call) ? value.call : value
|
53
|
+
next unless @delegate.command_args.join(' ') =~ pattern
|
54
|
+
until io.eof?
|
55
|
+
line = io.gets
|
56
|
+
line_no ||= 0
|
57
|
+
@delegate.handle_stdout(line.chomp, line_no) if @delegate.respond_to?(:handle_stdout)
|
58
|
+
line_no += 1
|
60
59
|
end
|
60
|
+
return value.respond_to?(:call) ? value.call : value
|
61
61
|
end
|
62
62
|
|
63
63
|
if @delegate.respond_to?(:around_execute)
|
64
64
|
@delegate.around_execute(&block)
|
65
65
|
else
|
66
|
-
|
66
|
+
yield
|
67
67
|
end
|
68
68
|
end
|
69
69
|
end
|
@@ -82,11 +82,11 @@ module Masamune::MockCommand
|
|
82
82
|
end
|
83
83
|
|
84
84
|
def mock_success
|
85
|
-
OpenStruct.new(
|
85
|
+
OpenStruct.new(success?: true)
|
86
86
|
end
|
87
87
|
|
88
88
|
def mock_failure
|
89
|
-
OpenStruct.new(
|
89
|
+
OpenStruct.new(success?: false)
|
90
90
|
end
|
91
91
|
|
92
92
|
def mock_command(pattern, value = nil, io = StringIO.new, &block)
|
@@ -38,11 +38,11 @@ class Masamune::MockDelegate
|
|
38
38
|
end
|
39
39
|
|
40
40
|
def handle_stdout(line, line_no)
|
41
|
-
|
41
|
+
stdout[line_no] = line
|
42
42
|
end
|
43
43
|
|
44
44
|
def handle_stderr(line, line_no)
|
45
|
-
|
45
|
+
stderr[line_no] = line
|
46
46
|
end
|
47
47
|
|
48
48
|
def handle_failure(code)
|
@@ -63,7 +63,7 @@ class Masamune::MockFilesystem < Delegator
|
|
63
63
|
def glob_stat(pattern, options = {})
|
64
64
|
return Set.new(to_enum(:glob_stat, pattern, options)) unless block_given?
|
65
65
|
file_regexp = glob_to_regexp(pattern, recursive: true)
|
66
|
-
@files.each do |
|
66
|
+
@files.each do |_name, stat|
|
67
67
|
next if stat.name == dirname(pattern)
|
68
68
|
next unless stat.name =~ file_regexp
|
69
69
|
yield stat
|
@@ -85,12 +85,12 @@ class Masamune::MockFilesystem < Delegator
|
|
85
85
|
def clear!
|
86
86
|
end
|
87
87
|
|
88
|
-
def check_immutable_path!(
|
88
|
+
def check_immutable_path!(_file)
|
89
89
|
true
|
90
90
|
end
|
91
91
|
|
92
92
|
[:mkdir!, :copy_file_to_file, :copy_file_to_dir, :copy_dir, :remove_file, :remove_dir, :move_file_to_file, :move_file_to_dir, :move_dir, :write].each do |method|
|
93
|
-
define_method(method) do |*
|
93
|
+
define_method(method) do |*_args|
|
94
94
|
# Empty
|
95
95
|
end
|
96
96
|
end
|
@@ -29,20 +29,22 @@ module Masamune::SharedExampleGroup
|
|
29
29
|
io.sync = true
|
30
30
|
io.write stdin
|
31
31
|
io.close_write
|
32
|
-
|
33
|
-
stdout << io.read
|
34
|
-
end
|
32
|
+
stdout << io.read until io.eof?
|
35
33
|
end
|
36
34
|
stdout.string
|
37
35
|
end
|
38
36
|
|
39
37
|
def capture_output
|
40
|
-
@stdout
|
41
|
-
|
42
|
-
|
38
|
+
@stdout = StringIO.new
|
39
|
+
@stderr = StringIO.new
|
40
|
+
tmp_stdout = $stdout
|
41
|
+
$stdout = @stdout
|
42
|
+
tmp_stderr = $stderr
|
43
|
+
$stderr = @stderr
|
43
44
|
yield
|
44
45
|
ensure
|
45
|
-
$stdout
|
46
|
+
$stdout = tmp_stdout
|
47
|
+
$stderr = tmp_stderr
|
46
48
|
end
|
47
49
|
|
48
50
|
def capture(enable = true)
|
@@ -85,25 +87,21 @@ module Masamune::SharedExampleGroup
|
|
85
87
|
end
|
86
88
|
end
|
87
89
|
|
88
|
-
# TODO encapsulate commands as runners
|
90
|
+
# TODO: encapsulate commands as runners
|
89
91
|
def setup_example_input!(fixture)
|
90
92
|
fixture.inputs.each do |input|
|
91
93
|
if input['file']
|
92
94
|
filesystem.write(with_delim(input['data'], input['delim']), input['file'])
|
93
95
|
end
|
94
96
|
|
95
|
-
if input['hive']
|
96
|
-
hive(exec: input['hive'])
|
97
|
-
end
|
97
|
+
hive(exec: input['hive']) if input['hive']
|
98
98
|
|
99
|
-
if input['psql']
|
100
|
-
postgres(exec: input['psql'])
|
101
|
-
end
|
99
|
+
postgres(exec: input['psql']) if input['psql']
|
102
100
|
end
|
103
101
|
end
|
104
102
|
|
105
103
|
def gather_example_output(fixture)
|
106
|
-
|
104
|
+
raise "No outputs defined for #{fixture.file_name}" if fixture.outputs.none?
|
107
105
|
fixture.outputs.each do |output|
|
108
106
|
output_file = output['file'] || Tempfile.new('masamune').path
|
109
107
|
|
@@ -123,7 +121,7 @@ module Masamune::SharedExampleGroup
|
|
123
121
|
if output['hive'] && output['hive'].is_a?(String)
|
124
122
|
hive(exec: output['hive'], output: output_file)
|
125
123
|
elsif output['table']
|
126
|
-
table = eval "catalog.#{output['table']}"
|
124
|
+
table = eval "catalog.#{output['table']}" # rubocop:disable Lint/Eval
|
127
125
|
query = denormalize_table(table, output.slice('columns', 'order', 'except', 'include')).to_s
|
128
126
|
# FIXME: define format based on table.store.format
|
129
127
|
case table.store.type
|
@@ -32,7 +32,7 @@ module Masamune::StepExampleGroup
|
|
32
32
|
end
|
33
33
|
|
34
34
|
context "with #{fixture_file} fixture" do
|
35
|
-
let(:fixture) { Masamune::StepFixture.load({file: fixture_file}, binding) }
|
35
|
+
let(:fixture) { Masamune::StepFixture.load({ file: fixture_file }, binding) }
|
36
36
|
|
37
37
|
let(:input) { fixture.input }
|
38
38
|
let(:output) { fixture.output }
|
@@ -62,7 +62,7 @@ module Masamune::StepExampleGroup
|
|
62
62
|
end
|
63
63
|
|
64
64
|
RSpec.configure do |config|
|
65
|
-
config.include Masamune::StepExampleGroup, :
|
66
|
-
config.include Masamune::StepExampleGroup, :
|
67
|
-
config.include Masamune::StepExampleGroup, :
|
65
|
+
config.include Masamune::StepExampleGroup, type: :step, file_path: %r{.*/spec/.*step_spec\.rb}
|
66
|
+
config.include Masamune::StepExampleGroup, type: :step, file_path: %r{.*/spec/.*mapper_spec\.rb}
|
67
|
+
config.include Masamune::StepExampleGroup, type: :step, file_path: %r{.*/spec/.*reducer_spec\.rb}
|
68
68
|
end
|
@@ -38,7 +38,7 @@ module Masamune
|
|
38
38
|
class << self
|
39
39
|
def load(options = {}, context = binding)
|
40
40
|
file = options[:file]
|
41
|
-
raise ArgumentError, "Fixture '#{file}' does not exist" unless File.
|
41
|
+
raise ArgumentError, "Fixture '#{file}' does not exist" unless File.exist?(file)
|
42
42
|
YAML.load(ERB.new(File.read(file)).result(context)).tap do |data|
|
43
43
|
return new(options.merge(data: data))
|
44
44
|
end
|
@@ -56,5 +56,5 @@ module Masamune::TaskExampleGroup
|
|
56
56
|
end
|
57
57
|
|
58
58
|
RSpec.configure do |config|
|
59
|
-
config.include Masamune::TaskExampleGroup, :
|
59
|
+
config.include Masamune::TaskExampleGroup, type: :task, file_path: %r{.*/spec/.*task_spec\.rb}
|
60
60
|
end
|
@@ -25,8 +25,9 @@ module Masamune::ThorMute
|
|
25
25
|
def self.included(base)
|
26
26
|
base.instance_eval do
|
27
27
|
def create_command(*a)
|
28
|
-
|
29
|
-
|
28
|
+
tmp_stdout = $stdout
|
29
|
+
$stdout = StringIO.new
|
30
|
+
super(*a)
|
30
31
|
ensure
|
31
32
|
$stdout = tmp_stdout
|
32
33
|
end
|
@@ -21,12 +21,15 @@
|
|
21
21
|
# THE SOFTWARE.
|
22
22
|
|
23
23
|
module TaskExampleGroup
|
24
|
-
def capture(stdout, stderr
|
25
|
-
tmp_stdout
|
26
|
-
|
24
|
+
def capture(stdout, stderr)
|
25
|
+
tmp_stdout = $stdout
|
26
|
+
$stdout = stdout
|
27
|
+
tmp_stderr = $stderr
|
28
|
+
$stderr = stderr
|
27
29
|
yield
|
28
30
|
ensure
|
29
|
-
$stdout
|
31
|
+
$stdout = tmp_stdout
|
32
|
+
$stderr = tmp_stderr
|
30
33
|
end
|
31
34
|
|
32
35
|
shared_examples 'general usage' do
|
@@ -32,5 +32,5 @@ module TransformExampleGroup
|
|
32
32
|
end
|
33
33
|
|
34
34
|
RSpec.configure do |config|
|
35
|
-
config.include TransformExampleGroup, :
|
35
|
+
config.include TransformExampleGroup, type: :action, file_path: %r{spec/masamune/transform}
|
36
36
|
end
|
@@ -31,7 +31,7 @@ shared_examples_for Masamune::Commands::PostgresCommon do
|
|
31
31
|
end
|
32
32
|
|
33
33
|
context 'with pgpass_file' do
|
34
|
-
let(:configuration) { {:
|
34
|
+
let(:configuration) { { pgpass_file: 'pgpass_file' } }
|
35
35
|
|
36
36
|
before do
|
37
37
|
allow(File).to receive(:readable?) { true }
|
@@ -41,7 +41,7 @@ shared_examples_for Masamune::Commands::PostgresCommon do
|
|
41
41
|
end
|
42
42
|
|
43
43
|
context 'with pgpass_file that is not readable' do
|
44
|
-
let(:configuration) { {:
|
44
|
+
let(:configuration) { { pgpass_file: 'pgpass_file' } }
|
45
45
|
|
46
46
|
before do
|
47
47
|
allow(File).to receive(:readable?) { false }
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: masamune
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.17.
|
4
|
+
version: 0.17.13
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Michael Andrews
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-05-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: thor
|
@@ -150,6 +150,20 @@ dependencies:
|
|
150
150
|
- - ">="
|
151
151
|
- !ruby/object:Gem::Version
|
152
152
|
version: '0.9'
|
153
|
+
- !ruby/object:Gem::Dependency
|
154
|
+
name: rubocop
|
155
|
+
requirement: !ruby/object:Gem::Requirement
|
156
|
+
requirements:
|
157
|
+
- - ">="
|
158
|
+
- !ruby/object:Gem::Version
|
159
|
+
version: '0'
|
160
|
+
type: :development
|
161
|
+
prerelease: false
|
162
|
+
version_requirements: !ruby/object:Gem::Requirement
|
163
|
+
requirements:
|
164
|
+
- - ">="
|
165
|
+
- !ruby/object:Gem::Version
|
166
|
+
version: '0'
|
153
167
|
- !ruby/object:Gem::Dependency
|
154
168
|
name: user_agent_parser
|
155
169
|
requirement: !ruby/object:Gem::Requirement
|