masamune 0.17.12 → 0.17.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Rakefile +3 -3
- data/bin/masamune-aws-emr +1 -1
- data/bin/masamune-dump +1 -1
- data/bin/masamune-hive +1 -1
- data/bin/masamune-psql +1 -1
- data/bin/masamune-shell +1 -1
- data/lib/masamune.rb +1 -1
- data/lib/masamune/actions/aws_emr.rb +2 -2
- data/lib/masamune/actions/data_flow.rb +10 -10
- data/lib/masamune/actions/date_parse.rb +4 -6
- data/lib/masamune/actions/execute.rb +3 -3
- data/lib/masamune/actions/filesystem.rb +1 -1
- data/lib/masamune/actions/hadoop_filesystem.rb +2 -2
- data/lib/masamune/actions/hive.rb +5 -5
- data/lib/masamune/actions/invoke_parallel.rb +5 -5
- data/lib/masamune/actions/postgres.rb +6 -6
- data/lib/masamune/actions/postgres_admin.rb +1 -1
- data/lib/masamune/actions/s3cmd.rb +3 -3
- data/lib/masamune/actions/transform.rb +4 -4
- data/lib/masamune/after_initialize_callbacks.rb +4 -4
- data/lib/masamune/cached_filesystem.rb +7 -11
- data/lib/masamune/commands/aws_emr.rb +12 -12
- data/lib/masamune/commands/hadoop_filesystem.rb +8 -8
- data/lib/masamune/commands/hadoop_streaming.rb +16 -16
- data/lib/masamune/commands/hive.rb +31 -35
- data/lib/masamune/commands/postgres.rb +30 -32
- data/lib/masamune/commands/postgres_admin.rb +17 -16
- data/lib/masamune/commands/postgres_common.rb +1 -1
- data/lib/masamune/commands/retry_with_backoff.rb +19 -19
- data/lib/masamune/commands/s3cmd.rb +7 -7
- data/lib/masamune/commands/shell.rb +25 -38
- data/lib/masamune/configuration.rb +8 -9
- data/lib/masamune/data_plan/builder.rb +5 -5
- data/lib/masamune/data_plan/elem.rb +7 -7
- data/lib/masamune/data_plan/engine.rb +12 -15
- data/lib/masamune/data_plan/rule.rb +14 -12
- data/lib/masamune/data_plan/set.rb +13 -13
- data/lib/masamune/environment.rb +3 -3
- data/lib/masamune/filesystem.rb +29 -28
- data/lib/masamune/helpers/postgres.rb +8 -5
- data/lib/masamune/io.rb +2 -2
- data/lib/masamune/last_element.rb +3 -3
- data/lib/masamune/method_logger.rb +1 -1
- data/lib/masamune/schema/catalog.rb +9 -12
- data/lib/masamune/schema/column.rb +12 -10
- data/lib/masamune/schema/dimension.rb +5 -4
- data/lib/masamune/schema/fact.rb +2 -2
- data/lib/masamune/schema/map.rb +9 -13
- data/lib/masamune/schema/row.rb +3 -2
- data/lib/masamune/schema/store.rb +13 -14
- data/lib/masamune/schema/table.rb +24 -28
- data/lib/masamune/schema/table_reference.rb +7 -7
- data/lib/masamune/spec_helper.rb +1 -1
- data/lib/masamune/string_format.rb +1 -1
- data/lib/masamune/tasks/aws_emr_thor.rb +9 -9
- data/lib/masamune/tasks/dump_thor.rb +5 -5
- data/lib/masamune/tasks/hive_thor.rb +13 -13
- data/lib/masamune/tasks/postgres_thor.rb +8 -8
- data/lib/masamune/tasks/shell_thor.rb +3 -3
- data/lib/masamune/template.rb +4 -4
- data/lib/masamune/thor.rb +26 -25
- data/lib/masamune/transform/common/denormalize_table.rb +6 -6
- data/lib/masamune/transform/define_table.rb +1 -1
- data/lib/masamune/transform/hive/define_table.rb +0 -2
- data/lib/masamune/transform/insert_reference_values.rb +1 -1
- data/lib/masamune/transform/operator.rb +3 -2
- data/lib/masamune/transform/postgres/bulk_upsert.rb +0 -3
- data/lib/masamune/transform/postgres/deduplicate_dimension.rb +4 -6
- data/lib/masamune/transform/postgres/define_table.rb +2 -2
- data/lib/masamune/transform/postgres/insert_reference_values.rb +1 -4
- data/lib/masamune/transform/postgres/relabel_dimension.rb +1 -3
- data/lib/masamune/transform/postgres/rollup_fact.rb +3 -5
- data/lib/masamune/transform/postgres/snapshot_dimension.rb +1 -4
- data/lib/masamune/transform/postgres/stage_dimension.rb +8 -10
- data/lib/masamune/transform/postgres/stage_fact.rb +15 -14
- data/lib/masamune/version.rb +1 -1
- data/spec/masamune/actions/aws_emr_spec.rb +8 -8
- data/spec/masamune/actions/execute_spec.rb +6 -6
- data/spec/masamune/actions/hive_spec.rb +20 -15
- data/spec/masamune/actions/postgres_spec.rb +17 -10
- data/spec/masamune/after_initialization_callbacks_spec.rb +13 -7
- data/spec/masamune/commands/aws_emr_spec.rb +10 -10
- data/spec/masamune/commands/hadoop_filesystem_spec.rb +3 -3
- data/spec/masamune/commands/hadoop_streaming_spec.rb +4 -4
- data/spec/masamune/commands/hive_spec.rb +11 -11
- data/spec/masamune/commands/postgres_admin_spec.rb +9 -9
- data/spec/masamune/commands/postgres_spec.rb +15 -15
- data/spec/masamune/commands/retry_with_backoff_spec.rb +10 -7
- data/spec/masamune/commands/s3cmd_spec.rb +3 -3
- data/spec/masamune/commands/shell_spec.rb +16 -16
- data/spec/masamune/data_plan/builder_spec.rb +22 -19
- data/spec/masamune/data_plan/elem_spec.rb +20 -20
- data/spec/masamune/data_plan/engine_spec.rb +41 -33
- data/spec/masamune/data_plan/rule_spec.rb +19 -19
- data/spec/masamune/data_plan/set_spec.rb +8 -6
- data/spec/masamune/environment_spec.rb +5 -5
- data/spec/masamune/filesystem_spec.rb +87 -87
- data/spec/masamune/helpers/postgres_spec.rb +1 -1
- data/spec/masamune/rspec/job_fixture_spec.rb +0 -1
- data/spec/masamune/schema/catalog_spec.rb +16 -16
- data/spec/masamune/schema/column_spec.rb +19 -19
- data/spec/masamune/schema/dimension_spec.rb +2 -2
- data/spec/masamune/schema/fact_spec.rb +3 -3
- data/spec/masamune/schema/map_spec.rb +41 -41
- data/spec/masamune/schema/table_spec.rb +16 -16
- data/spec/masamune/string_format_spec.rb +7 -7
- data/spec/masamune/tasks/dump_thor_spec.rb +6 -6
- data/spec/masamune/tasks/hive_thor_spec.rb +2 -2
- data/spec/masamune/template_spec.rb +3 -2
- data/spec/masamune/thor_spec.rb +8 -8
- data/spec/masamune/transform/bulk_upsert.dimension_spec.rb +3 -3
- data/spec/masamune/transform/define_table.dimension_spec.rb +3 -3
- data/spec/masamune/transform/define_table.fact_spec.rb +5 -5
- data/spec/masamune/transform/define_table.table_spec.rb +6 -6
- data/spec/masamune/transform/denormalize_table_spec.rb +15 -15
- data/spec/masamune/transform/insert_reference_values.dimension_spec.rb +1 -1
- data/spec/masamune/transform/load_dimension_spec.rb +1 -1
- data/spec/masamune/transform/load_fact_spec.rb +1 -1
- data/spec/masamune/transform/rollup_fact_spec.rb +4 -4
- data/spec/masamune/transform/stage_dimension_spec.rb +3 -3
- data/spec/masamune/transform/stage_fact_spec.rb +4 -4
- data/spec/spec_helper.rb +1 -1
- data/spec/support/masamune/example_group.rb +2 -2
- data/spec/support/masamune/job_example_group.rb +3 -3
- data/spec/support/masamune/job_fixture.rb +4 -4
- data/spec/support/masamune/mock_command.rb +10 -10
- data/spec/support/masamune/mock_delegate.rb +2 -2
- data/spec/support/masamune/mock_filesystem.rb +3 -3
- data/spec/support/masamune/shared_example_group.rb +14 -16
- data/spec/support/masamune/step_example_group.rb +4 -4
- data/spec/support/masamune/step_fixture.rb +1 -1
- data/spec/support/masamune/task_example_group.rb +1 -1
- data/spec/support/masamune/thor_mute.rb +3 -2
- data/spec/support/rspec/example/action_example_group.rb +1 -1
- data/spec/support/rspec/example/task_example_group.rb +7 -4
- data/spec/support/rspec/example/transform_example_group.rb +1 -1
- data/spec/support/shared_examples/postgres_common_examples.rb +2 -2
- metadata +16 -2
@@ -69,7 +69,7 @@ describe Masamune::Transform::LoadFact do
|
|
69
69
|
end
|
70
70
|
|
71
71
|
let(:files) { (1..3).map { |i| double(path: "output_#{i}.csv") } }
|
72
|
-
let(:date) { DateTime.civil(2014,8) }
|
72
|
+
let(:date) { DateTime.civil(2014, 8) }
|
73
73
|
let(:target) { catalog.postgres.visits_fact }
|
74
74
|
let(:source) { catalog.postgres.visits_file }
|
75
75
|
let(:source_table) { source.stage_table(suffix: 'file', table: target, inherit: false) }
|
@@ -29,7 +29,7 @@ describe Masamune::Transform::RollupFact do
|
|
29
29
|
column 'id', type: :sequence, surrogate_key: true, auto: true
|
30
30
|
column 'name', type: :string
|
31
31
|
|
32
|
-
row name: 'current_database()', attributes: {default: true}
|
32
|
+
row name: 'current_database()', attributes: { default: true }
|
33
33
|
end
|
34
34
|
|
35
35
|
dimension 'date', type: :date do
|
@@ -82,7 +82,7 @@ describe Masamune::Transform::RollupFact do
|
|
82
82
|
end
|
83
83
|
|
84
84
|
context 'with postgres transaction fact' do
|
85
|
-
let(:date) { DateTime.civil(2014,8) }
|
85
|
+
let(:date) { DateTime.civil(2014, 8) }
|
86
86
|
let(:source) { catalog.postgres.visits_transaction_fact }
|
87
87
|
let(:target) { catalog.postgres.visits_hourly_fact }
|
88
88
|
|
@@ -180,7 +180,7 @@ describe Masamune::Transform::RollupFact do
|
|
180
180
|
end
|
181
181
|
|
182
182
|
context 'with postgres hourly fact' do
|
183
|
-
let(:date) { DateTime.civil(2014,8) }
|
183
|
+
let(:date) { DateTime.civil(2014, 8) }
|
184
184
|
let(:source) { catalog.postgres.visits_hourly_fact }
|
185
185
|
let(:target) { catalog.postgres.visits_daily_fact }
|
186
186
|
|
@@ -278,7 +278,7 @@ describe Masamune::Transform::RollupFact do
|
|
278
278
|
end
|
279
279
|
|
280
280
|
context 'with postgres daily fact' do
|
281
|
-
let(:date) { DateTime.civil(2014,8) }
|
281
|
+
let(:date) { DateTime.civil(2014, 8) }
|
282
282
|
let(:source) { catalog.postgres.visits_daily_fact }
|
283
283
|
let(:target) { catalog.postgres.visits_monthly_fact }
|
284
284
|
|
@@ -26,14 +26,14 @@ describe Masamune::Transform::StageDimension do
|
|
26
26
|
dimension 'cluster', type: :mini do
|
27
27
|
column 'id', type: :integer, surrogate_key: true, auto: true
|
28
28
|
column 'name', type: :string, unique: true
|
29
|
-
row name: 'default', attributes: {default: true}
|
29
|
+
row name: 'default', attributes: { default: true }
|
30
30
|
end
|
31
31
|
|
32
32
|
dimension 'user_account_state', type: :mini do
|
33
33
|
column 'name', type: :string, unique: true
|
34
34
|
column 'description', type: :string
|
35
35
|
row name: 'registered', description: 'Registered'
|
36
|
-
row name: 'active', description: 'Active', attributes: {default: true}
|
36
|
+
row name: 'active', description: 'Active', attributes: { default: true }
|
37
37
|
row name: 'inactive', description: 'Inactive'
|
38
38
|
end
|
39
39
|
|
@@ -41,7 +41,7 @@ describe Masamune::Transform::StageDimension do
|
|
41
41
|
references :cluster
|
42
42
|
column 'tenant_id', type: :integer, unique: true, natural_key: true
|
43
43
|
column 'department_id', type: :integer, unique: true, natural_key: true
|
44
|
-
row tenant_id: -1, department_id: -1, attributes: {default: true}
|
44
|
+
row tenant_id: -1, department_id: -1, attributes: { default: true }
|
45
45
|
end
|
46
46
|
|
47
47
|
dimension 'user', type: :four do
|
@@ -29,7 +29,7 @@ describe Masamune::Transform::StageFact do
|
|
29
29
|
column 'id', type: :sequence, surrogate_key: true, auto: true
|
30
30
|
column 'name', type: :string
|
31
31
|
|
32
|
-
row name: 'current_database()', attributes: {default: true}
|
32
|
+
row name: 'current_database()', attributes: { default: true }
|
33
33
|
end
|
34
34
|
|
35
35
|
dimension 'date', type: :date do
|
@@ -68,8 +68,8 @@ describe Masamune::Transform::StageFact do
|
|
68
68
|
column 'tenant_id', type: :integer, natural_key: true
|
69
69
|
column 'group_id', type: :integer, natural_key: true
|
70
70
|
column 'group_mode', type: :enum, sub_type: 'group_mode', values: %(missing unknown public private), index: true, natural_key: true, default: 'missing'
|
71
|
-
row group_id: -1, group_mode: 'missing', attributes: {id: :missing}
|
72
|
-
row group_id: -2, group_mode: 'unknown', attributes: {id: :unknown}
|
71
|
+
row group_id: -1, group_mode: 'missing', attributes: { id: :missing }
|
72
|
+
row group_id: -2, group_mode: 'unknown', attributes: { id: :unknown }
|
73
73
|
end
|
74
74
|
|
75
75
|
fact 'visits', partition: 'y%Ym%m', grain: %w(hourly daily monthly) do
|
@@ -104,7 +104,7 @@ describe Masamune::Transform::StageFact do
|
|
104
104
|
end
|
105
105
|
end
|
106
106
|
|
107
|
-
let(:date) { DateTime.civil(2014,8) }
|
107
|
+
let(:date) { DateTime.civil(2014, 8) }
|
108
108
|
let(:target) { catalog.postgres.visits_hourly_fact }
|
109
109
|
let(:source) { catalog.postgres.visits_hourly_file.stage_table(suffix: 'file', table: target, inherit: false) }
|
110
110
|
|
data/spec/spec_helper.rb
CHANGED
@@ -26,7 +26,7 @@ Bundler.require(:default, :development, :test)
|
|
26
26
|
require 'tempfile'
|
27
27
|
require 'tmpdir'
|
28
28
|
|
29
|
-
Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f}
|
29
|
+
Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each { |f| require f }
|
30
30
|
|
31
31
|
require 'active_support/core_ext/string/strip'
|
32
32
|
|
@@ -25,11 +25,11 @@ require 'masamune/has_environment'
|
|
25
25
|
# Separate environment for test harness itself
|
26
26
|
module Masamune::ExampleGroup
|
27
27
|
include Masamune::HasEnvironment
|
28
|
-
extend self
|
28
|
+
extend self # rubocop:disable Style/ModuleFunction
|
29
29
|
|
30
30
|
def self.included(base)
|
31
31
|
base.before(:all) do
|
32
|
-
|
32
|
+
filesystem.environment = self.environment = Masamune::ExampleGroup.environment
|
33
33
|
Thor.send(:include, Masamune::ThorMute)
|
34
34
|
end
|
35
35
|
end
|
@@ -38,7 +38,7 @@ module Masamune::JobExampleGroup
|
|
38
38
|
it "should match #{fixture_file}" do
|
39
39
|
aggregate_failures 'generates expected output' do
|
40
40
|
gather_example_output(fixture) do |actual_data, expect_file, expect_data|
|
41
|
-
expect(File.
|
41
|
+
expect(File.exist?(expect_file)).to eq(true)
|
42
42
|
expect(actual_data).to eq(expect_data)
|
43
43
|
end
|
44
44
|
end
|
@@ -57,6 +57,6 @@ module Masamune::JobExampleGroup
|
|
57
57
|
end
|
58
58
|
|
59
59
|
RSpec.configure do |config|
|
60
|
-
config.include Masamune::JobExampleGroup, :
|
61
|
-
config.include Masamune::JobExampleGroup, :
|
60
|
+
config.include Masamune::JobExampleGroup, type: :job, file_path: %r{.*/spec/.*job_spec\.rb}
|
61
|
+
config.include Masamune::JobExampleGroup, type: :task, file_path: %r{.*/spec/.*task_spec\.rb}
|
62
62
|
end
|
@@ -43,7 +43,7 @@ module Masamune
|
|
43
43
|
class << self
|
44
44
|
def load(options = {}, context = binding)
|
45
45
|
file = file_name(options)
|
46
|
-
raise ArgumentError, "Fixture '#{file}' does not exist" unless File.
|
46
|
+
raise ArgumentError, "Fixture '#{file}' does not exist" unless File.exist?(file)
|
47
47
|
YAML.load(ERB.new(File.read(file)).result(context)).tap do |data|
|
48
48
|
return new(options.merge(data: data, context: context))
|
49
49
|
end
|
@@ -77,7 +77,7 @@ module Masamune
|
|
77
77
|
@data['inputs'].each do |input|
|
78
78
|
file.puts '-'.indent(INDENT)
|
79
79
|
serialize(input) do |elem|
|
80
|
-
file.puts elem.indent(INDENT*2)
|
80
|
+
file.puts elem.indent(INDENT * 2)
|
81
81
|
end
|
82
82
|
end
|
83
83
|
file.puts
|
@@ -85,7 +85,7 @@ module Masamune
|
|
85
85
|
@data['outputs'].each do |output|
|
86
86
|
file.puts '-'.indent(INDENT)
|
87
87
|
serialize(output) do |elem|
|
88
|
-
file.puts elem.indent(INDENT*2)
|
88
|
+
file.puts elem.indent(INDENT * 2)
|
89
89
|
end
|
90
90
|
end
|
91
91
|
end
|
@@ -97,7 +97,7 @@ module Masamune
|
|
97
97
|
@data['inputs'].map do |input|
|
98
98
|
if input['reference']
|
99
99
|
raise ArgumentError, "reference in #{file_name} requires fixture" unless input['reference']['fixture'] || input['reference']['file']
|
100
|
-
reference = self.class.load({path: input['reference']['path'] || path, name: input['reference']['fixture'], file: input['reference']['file'], type: @type}, @context)
|
100
|
+
reference = self.class.load({ path: input['reference']['path'] || path, name: input['reference']['fixture'], file: input['reference']['file'], type: @type }, @context)
|
101
101
|
section = input['reference']['section'] || 'outputs'
|
102
102
|
reference.send(section) if reference.respond_to?(section)
|
103
103
|
else
|
@@ -50,20 +50,20 @@ module Masamune::MockCommand
|
|
50
50
|
|
51
51
|
def around_execute(&block)
|
52
52
|
self.class.patterns.each do |pattern, (value, io)|
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
return value.respond_to?(:call) ? value.call : value
|
53
|
+
next unless @delegate.command_args.join(' ') =~ pattern
|
54
|
+
until io.eof?
|
55
|
+
line = io.gets
|
56
|
+
line_no ||= 0
|
57
|
+
@delegate.handle_stdout(line.chomp, line_no) if @delegate.respond_to?(:handle_stdout)
|
58
|
+
line_no += 1
|
60
59
|
end
|
60
|
+
return value.respond_to?(:call) ? value.call : value
|
61
61
|
end
|
62
62
|
|
63
63
|
if @delegate.respond_to?(:around_execute)
|
64
64
|
@delegate.around_execute(&block)
|
65
65
|
else
|
66
|
-
|
66
|
+
yield
|
67
67
|
end
|
68
68
|
end
|
69
69
|
end
|
@@ -82,11 +82,11 @@ module Masamune::MockCommand
|
|
82
82
|
end
|
83
83
|
|
84
84
|
def mock_success
|
85
|
-
OpenStruct.new(
|
85
|
+
OpenStruct.new(success?: true)
|
86
86
|
end
|
87
87
|
|
88
88
|
def mock_failure
|
89
|
-
OpenStruct.new(
|
89
|
+
OpenStruct.new(success?: false)
|
90
90
|
end
|
91
91
|
|
92
92
|
def mock_command(pattern, value = nil, io = StringIO.new, &block)
|
@@ -38,11 +38,11 @@ class Masamune::MockDelegate
|
|
38
38
|
end
|
39
39
|
|
40
40
|
def handle_stdout(line, line_no)
|
41
|
-
|
41
|
+
stdout[line_no] = line
|
42
42
|
end
|
43
43
|
|
44
44
|
def handle_stderr(line, line_no)
|
45
|
-
|
45
|
+
stderr[line_no] = line
|
46
46
|
end
|
47
47
|
|
48
48
|
def handle_failure(code)
|
@@ -63,7 +63,7 @@ class Masamune::MockFilesystem < Delegator
|
|
63
63
|
def glob_stat(pattern, options = {})
|
64
64
|
return Set.new(to_enum(:glob_stat, pattern, options)) unless block_given?
|
65
65
|
file_regexp = glob_to_regexp(pattern, recursive: true)
|
66
|
-
@files.each do |
|
66
|
+
@files.each do |_name, stat|
|
67
67
|
next if stat.name == dirname(pattern)
|
68
68
|
next unless stat.name =~ file_regexp
|
69
69
|
yield stat
|
@@ -85,12 +85,12 @@ class Masamune::MockFilesystem < Delegator
|
|
85
85
|
def clear!
|
86
86
|
end
|
87
87
|
|
88
|
-
def check_immutable_path!(
|
88
|
+
def check_immutable_path!(_file)
|
89
89
|
true
|
90
90
|
end
|
91
91
|
|
92
92
|
[:mkdir!, :copy_file_to_file, :copy_file_to_dir, :copy_dir, :remove_file, :remove_dir, :move_file_to_file, :move_file_to_dir, :move_dir, :write].each do |method|
|
93
|
-
define_method(method) do |*
|
93
|
+
define_method(method) do |*_args|
|
94
94
|
# Empty
|
95
95
|
end
|
96
96
|
end
|
@@ -29,20 +29,22 @@ module Masamune::SharedExampleGroup
|
|
29
29
|
io.sync = true
|
30
30
|
io.write stdin
|
31
31
|
io.close_write
|
32
|
-
|
33
|
-
stdout << io.read
|
34
|
-
end
|
32
|
+
stdout << io.read until io.eof?
|
35
33
|
end
|
36
34
|
stdout.string
|
37
35
|
end
|
38
36
|
|
39
37
|
def capture_output
|
40
|
-
@stdout
|
41
|
-
|
42
|
-
|
38
|
+
@stdout = StringIO.new
|
39
|
+
@stderr = StringIO.new
|
40
|
+
tmp_stdout = $stdout
|
41
|
+
$stdout = @stdout
|
42
|
+
tmp_stderr = $stderr
|
43
|
+
$stderr = @stderr
|
43
44
|
yield
|
44
45
|
ensure
|
45
|
-
$stdout
|
46
|
+
$stdout = tmp_stdout
|
47
|
+
$stderr = tmp_stderr
|
46
48
|
end
|
47
49
|
|
48
50
|
def capture(enable = true)
|
@@ -85,25 +87,21 @@ module Masamune::SharedExampleGroup
|
|
85
87
|
end
|
86
88
|
end
|
87
89
|
|
88
|
-
# TODO encapsulate commands as runners
|
90
|
+
# TODO: encapsulate commands as runners
|
89
91
|
def setup_example_input!(fixture)
|
90
92
|
fixture.inputs.each do |input|
|
91
93
|
if input['file']
|
92
94
|
filesystem.write(with_delim(input['data'], input['delim']), input['file'])
|
93
95
|
end
|
94
96
|
|
95
|
-
if input['hive']
|
96
|
-
hive(exec: input['hive'])
|
97
|
-
end
|
97
|
+
hive(exec: input['hive']) if input['hive']
|
98
98
|
|
99
|
-
if input['psql']
|
100
|
-
postgres(exec: input['psql'])
|
101
|
-
end
|
99
|
+
postgres(exec: input['psql']) if input['psql']
|
102
100
|
end
|
103
101
|
end
|
104
102
|
|
105
103
|
def gather_example_output(fixture)
|
106
|
-
|
104
|
+
raise "No outputs defined for #{fixture.file_name}" if fixture.outputs.none?
|
107
105
|
fixture.outputs.each do |output|
|
108
106
|
output_file = output['file'] || Tempfile.new('masamune').path
|
109
107
|
|
@@ -123,7 +121,7 @@ module Masamune::SharedExampleGroup
|
|
123
121
|
if output['hive'] && output['hive'].is_a?(String)
|
124
122
|
hive(exec: output['hive'], output: output_file)
|
125
123
|
elsif output['table']
|
126
|
-
table = eval "catalog.#{output['table']}"
|
124
|
+
table = eval "catalog.#{output['table']}" # rubocop:disable Lint/Eval
|
127
125
|
query = denormalize_table(table, output.slice('columns', 'order', 'except', 'include')).to_s
|
128
126
|
# FIXME: define format based on table.store.format
|
129
127
|
case table.store.type
|
@@ -32,7 +32,7 @@ module Masamune::StepExampleGroup
|
|
32
32
|
end
|
33
33
|
|
34
34
|
context "with #{fixture_file} fixture" do
|
35
|
-
let(:fixture) { Masamune::StepFixture.load({file: fixture_file}, binding) }
|
35
|
+
let(:fixture) { Masamune::StepFixture.load({ file: fixture_file }, binding) }
|
36
36
|
|
37
37
|
let(:input) { fixture.input }
|
38
38
|
let(:output) { fixture.output }
|
@@ -62,7 +62,7 @@ module Masamune::StepExampleGroup
|
|
62
62
|
end
|
63
63
|
|
64
64
|
RSpec.configure do |config|
|
65
|
-
config.include Masamune::StepExampleGroup, :
|
66
|
-
config.include Masamune::StepExampleGroup, :
|
67
|
-
config.include Masamune::StepExampleGroup, :
|
65
|
+
config.include Masamune::StepExampleGroup, type: :step, file_path: %r{.*/spec/.*step_spec\.rb}
|
66
|
+
config.include Masamune::StepExampleGroup, type: :step, file_path: %r{.*/spec/.*mapper_spec\.rb}
|
67
|
+
config.include Masamune::StepExampleGroup, type: :step, file_path: %r{.*/spec/.*reducer_spec\.rb}
|
68
68
|
end
|
@@ -38,7 +38,7 @@ module Masamune
|
|
38
38
|
class << self
|
39
39
|
def load(options = {}, context = binding)
|
40
40
|
file = options[:file]
|
41
|
-
raise ArgumentError, "Fixture '#{file}' does not exist" unless File.
|
41
|
+
raise ArgumentError, "Fixture '#{file}' does not exist" unless File.exist?(file)
|
42
42
|
YAML.load(ERB.new(File.read(file)).result(context)).tap do |data|
|
43
43
|
return new(options.merge(data: data))
|
44
44
|
end
|
@@ -56,5 +56,5 @@ module Masamune::TaskExampleGroup
|
|
56
56
|
end
|
57
57
|
|
58
58
|
RSpec.configure do |config|
|
59
|
-
config.include Masamune::TaskExampleGroup, :
|
59
|
+
config.include Masamune::TaskExampleGroup, type: :task, file_path: %r{.*/spec/.*task_spec\.rb}
|
60
60
|
end
|
@@ -25,8 +25,9 @@ module Masamune::ThorMute
|
|
25
25
|
def self.included(base)
|
26
26
|
base.instance_eval do
|
27
27
|
def create_command(*a)
|
28
|
-
|
29
|
-
|
28
|
+
tmp_stdout = $stdout
|
29
|
+
$stdout = StringIO.new
|
30
|
+
super(*a)
|
30
31
|
ensure
|
31
32
|
$stdout = tmp_stdout
|
32
33
|
end
|
@@ -21,12 +21,15 @@
|
|
21
21
|
# THE SOFTWARE.
|
22
22
|
|
23
23
|
module TaskExampleGroup
|
24
|
-
def capture(stdout, stderr
|
25
|
-
tmp_stdout
|
26
|
-
|
24
|
+
def capture(stdout, stderr)
|
25
|
+
tmp_stdout = $stdout
|
26
|
+
$stdout = stdout
|
27
|
+
tmp_stderr = $stderr
|
28
|
+
$stderr = stderr
|
27
29
|
yield
|
28
30
|
ensure
|
29
|
-
$stdout
|
31
|
+
$stdout = tmp_stdout
|
32
|
+
$stderr = tmp_stderr
|
30
33
|
end
|
31
34
|
|
32
35
|
shared_examples 'general usage' do
|
@@ -32,5 +32,5 @@ module TransformExampleGroup
|
|
32
32
|
end
|
33
33
|
|
34
34
|
RSpec.configure do |config|
|
35
|
-
config.include TransformExampleGroup, :
|
35
|
+
config.include TransformExampleGroup, type: :action, file_path: %r{spec/masamune/transform}
|
36
36
|
end
|
@@ -31,7 +31,7 @@ shared_examples_for Masamune::Commands::PostgresCommon do
|
|
31
31
|
end
|
32
32
|
|
33
33
|
context 'with pgpass_file' do
|
34
|
-
let(:configuration) { {:
|
34
|
+
let(:configuration) { { pgpass_file: 'pgpass_file' } }
|
35
35
|
|
36
36
|
before do
|
37
37
|
allow(File).to receive(:readable?) { true }
|
@@ -41,7 +41,7 @@ shared_examples_for Masamune::Commands::PostgresCommon do
|
|
41
41
|
end
|
42
42
|
|
43
43
|
context 'with pgpass_file that is not readable' do
|
44
|
-
let(:configuration) { {:
|
44
|
+
let(:configuration) { { pgpass_file: 'pgpass_file' } }
|
45
45
|
|
46
46
|
before do
|
47
47
|
allow(File).to receive(:readable?) { false }
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: masamune
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.17.
|
4
|
+
version: 0.17.13
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Michael Andrews
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-
|
11
|
+
date: 2016-05-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: thor
|
@@ -150,6 +150,20 @@ dependencies:
|
|
150
150
|
- - ">="
|
151
151
|
- !ruby/object:Gem::Version
|
152
152
|
version: '0.9'
|
153
|
+
- !ruby/object:Gem::Dependency
|
154
|
+
name: rubocop
|
155
|
+
requirement: !ruby/object:Gem::Requirement
|
156
|
+
requirements:
|
157
|
+
- - ">="
|
158
|
+
- !ruby/object:Gem::Version
|
159
|
+
version: '0'
|
160
|
+
type: :development
|
161
|
+
prerelease: false
|
162
|
+
version_requirements: !ruby/object:Gem::Requirement
|
163
|
+
requirements:
|
164
|
+
- - ">="
|
165
|
+
- !ruby/object:Gem::Version
|
166
|
+
version: '0'
|
153
167
|
- !ruby/object:Gem::Dependency
|
154
168
|
name: user_agent_parser
|
155
169
|
requirement: !ruby/object:Gem::Requirement
|