masamune 0.14.0 → 0.15.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +10 -1
- data/Rakefile +37 -0
- data/bin/masamune-dump +22 -0
- data/bin/masamune-elastic-mapreduce +22 -0
- data/bin/masamune-hive +22 -0
- data/bin/masamune-psql +22 -0
- data/bin/masamune-shell +22 -0
- data/lib/masamune/cached_filesystem.rb +1 -1
- data/lib/masamune/commands/shell.rb +1 -1
- data/lib/masamune/configuration.rb +5 -7
- data/lib/masamune/data_plan/elem.rb +15 -7
- data/lib/masamune/data_plan/engine.rb +2 -0
- data/lib/masamune/data_plan/rule.rb +16 -5
- data/lib/masamune/data_plan/set.rb +8 -8
- data/lib/masamune/filesystem.rb +12 -6
- data/lib/masamune/schema/catalog.rb +6 -6
- data/lib/masamune/schema/column.rb +1 -1
- data/lib/masamune/schema/map.rb +6 -2
- data/lib/masamune/schema/store.rb +31 -3
- data/lib/masamune/tasks/shell_thor.rb +1 -1
- data/lib/masamune/thor.rb +12 -4
- data/lib/masamune/version.rb +1 -1
- data/lib/masamune.rb +0 -1
- data/spec/masamune/actions/elastic_mapreduce_spec.rb +0 -2
- data/spec/masamune/actions/execute_spec.rb +0 -2
- data/spec/masamune/actions/hadoop_filesystem_spec.rb +0 -2
- data/spec/masamune/actions/hadoop_streaming_spec.rb +0 -2
- data/spec/masamune/actions/hive_spec.rb +0 -2
- data/spec/masamune/actions/invoke_parallel_spec.rb +0 -2
- data/spec/masamune/actions/postgres_admin_spec.rb +0 -2
- data/spec/masamune/actions/postgres_spec.rb +0 -2
- data/spec/masamune/actions/s3cmd_spec.rb +0 -2
- data/spec/masamune/actions/transform_spec.rb +0 -2
- data/spec/masamune/after_initialization_callbacks_spec.rb +0 -2
- data/spec/masamune/cached_filesystem_spec.rb +0 -2
- data/spec/masamune/commands/hadoop_filesystem_spec.rb +0 -2
- data/spec/masamune/commands/hadoop_streaming_spec.rb +0 -2
- data/spec/masamune/commands/hive_spec.rb +0 -2
- data/spec/masamune/commands/postgres_admin_spec.rb +0 -2
- data/spec/masamune/commands/postgres_spec.rb +0 -2
- data/spec/masamune/commands/retry_with_backoff_spec.rb +0 -2
- data/spec/masamune/commands/s3cmd_spec.rb +0 -2
- data/spec/masamune/commands/shell_spec.rb +0 -2
- data/spec/masamune/configuration_spec.rb +12 -2
- data/spec/masamune/data_plan/builder_spec.rb +0 -2
- data/spec/masamune/data_plan/elem_spec.rb +73 -5
- data/spec/masamune/data_plan/engine_spec.rb +0 -2
- data/spec/masamune/data_plan/rule_spec.rb +51 -6
- data/spec/masamune/data_plan/set_spec.rb +2 -5
- data/spec/masamune/environment_spec.rb +0 -2
- data/spec/masamune/filesystem_spec.rb +33 -4
- data/spec/masamune/helpers/postgres_spec.rb +0 -2
- data/spec/masamune/rspec/job_fixture_spec.rb +365 -0
- data/spec/masamune/rspec/shared_example_group_spec.rb +73 -0
- data/spec/masamune/schema/catalog_spec.rb +14 -2
- data/spec/masamune/schema/column_spec.rb +0 -2
- data/spec/masamune/schema/dimension_spec.rb +0 -2
- data/spec/masamune/schema/fact_spec.rb +0 -2
- data/spec/masamune/schema/map_spec.rb +51 -2
- data/spec/masamune/schema/row_spec.rb +0 -2
- data/spec/masamune/schema/store_spec.rb +23 -2
- data/spec/masamune/schema/table_spec.rb +0 -2
- data/spec/masamune/string_format_spec.rb +0 -2
- data/spec/masamune/tasks/dump_thor_spec.rb +0 -3
- data/spec/masamune/tasks/elastic_mapreduce_thor_spec.rb +0 -3
- data/spec/masamune/tasks/hive_thor_spec.rb +0 -3
- data/spec/masamune/tasks/postgres_thor_spec.rb +0 -3
- data/spec/masamune/tasks/shell_thor_spec.rb +0 -3
- data/spec/masamune/template_spec.rb +0 -2
- data/spec/masamune/thor_spec.rb +53 -8
- data/spec/masamune/transform/bulk_upsert.dimension_spec.rb +0 -2
- data/spec/masamune/transform/consolidate_dimension_spec.rb +0 -2
- data/spec/masamune/transform/deduplicate_dimension_spec.rb +0 -2
- data/spec/masamune/transform/define_schema_spec.rb +0 -2
- data/spec/masamune/transform/define_table.dimension_spec.rb +0 -2
- data/spec/masamune/transform/define_table.fact_spec.rb +0 -2
- data/spec/masamune/transform/define_table.table_spec.rb +0 -2
- data/spec/masamune/transform/denormalize_table_spec.rb +0 -2
- data/spec/masamune/transform/insert_reference_values.dimension_spec.rb +0 -2
- data/spec/masamune/transform/insert_reference_values.fact_spec.rb +0 -2
- data/spec/masamune/transform/load_dimension_spec.rb +0 -2
- data/spec/masamune/transform/load_fact_spec.rb +0 -2
- data/spec/masamune/transform/relabel_dimension_spec.rb +0 -2
- data/spec/masamune/transform/rollup_fact_spec.rb +0 -2
- data/spec/masamune/transform/snapshot_dimension_spec.rb +0 -2
- data/spec/masamune/transform/stage_dimension_spec.rb +0 -2
- data/spec/masamune/transform/stage_fact_spec.rb +0 -2
- data/spec/masamune_spec.rb +0 -2
- data/spec/spec_helper.rb +2 -0
- data/spec/support/masamune/job_example_group.rb +62 -0
- data/spec/support/masamune/job_fixture.rb +137 -0
- data/spec/support/masamune/shared_example_group.rb +203 -0
- data/spec/support/masamune/step_example_group.rb +68 -0
- data/spec/support/masamune/step_fixture.rb +91 -0
- data/{lib/masamune/thor_loader.rb → spec/support/masamune/task_example_group.rb} +33 -10
- data/spec/support/rspec/example/action_example_group.rb +1 -1
- metadata +32 -3
data/lib/masamune/thor.rb
CHANGED
@@ -72,11 +72,14 @@ module Masamune
|
|
72
72
|
raise e unless %w(SIGHUP SIGTERM).include?(e.to_s)
|
73
73
|
instance.logger.debug("Exiting at user request on #{e.to_s}")
|
74
74
|
exit 0
|
75
|
+
rescue ::Thor::MalformattedArgumentError, ::Thor::RequiredArgumentMissingError => e
|
76
|
+
raise e
|
75
77
|
rescue => e
|
76
78
|
instance.logger.error("#{e.message} (#{e.class}) backtrace:")
|
77
79
|
e.backtrace.each { |x| instance.logger.error(x) }
|
80
|
+
$stderr.puts e.to_s
|
78
81
|
$stderr.puts "For complete debug log see: #{instance.log_file_name.to_s}"
|
79
|
-
|
82
|
+
exit 1
|
80
83
|
end
|
81
84
|
end
|
82
85
|
|
@@ -96,8 +99,7 @@ module Masamune
|
|
96
99
|
class_option :quiet, :type => :boolean, :aliases => '-q', :desc => 'Suppress all output', :default => false
|
97
100
|
class_option :verbose, :type => :boolean, :aliases => '-v', :desc => 'Print command execution information', :default => false
|
98
101
|
class_option :debug, :type => :boolean, :aliases => '-d', :desc => 'Print debugging information', :default => false
|
99
|
-
class_option :
|
100
|
-
class_option :dry_run, :type => :boolean, :aliases => '-n', :desc => 'Combination of --no-op and --verbose', :default => false
|
102
|
+
class_option :dry_run, :type => :boolean, :aliases => '-n', :desc => 'Do not execute commands that modify state', :default => false
|
101
103
|
class_option :config, :desc => 'Configuration file'
|
102
104
|
class_option :version, :desc => 'Print version and exit', :type => :boolean
|
103
105
|
class_option :lock, :desc => 'Optional job lock name', :type => :string
|
@@ -111,6 +113,8 @@ module Masamune
|
|
111
113
|
self.current_command_name = current_namespace ? current_namespace + ':' + current_task_name : current_task_name
|
112
114
|
self.class.instance = self
|
113
115
|
|
116
|
+
define_current_dir
|
117
|
+
|
114
118
|
if _options.is_a?(Array)
|
115
119
|
_options, self.extra = self.class.parse_extra(_options)
|
116
120
|
end
|
@@ -137,7 +141,6 @@ module Masamune
|
|
137
141
|
config.quiet = options[:quiet]
|
138
142
|
config.verbose = options[:verbose] || options[:dry_run]
|
139
143
|
config.debug = options[:debug]
|
140
|
-
config.no_op = options[:no_op] || options[:dry_run]
|
141
144
|
config.dry_run = options[:dry_run]
|
142
145
|
config.lock = options[:lock]
|
143
146
|
|
@@ -195,6 +198,11 @@ module Masamune
|
|
195
198
|
|
196
199
|
private
|
197
200
|
|
201
|
+
def define_current_dir
|
202
|
+
return unless current_task_name
|
203
|
+
filesystem.add_path(:current_dir, File.dirname(method(current_task_name).source_location.first))
|
204
|
+
end
|
205
|
+
|
198
206
|
def display_help?
|
199
207
|
options[:help] || current_task_name == 'help'
|
200
208
|
end
|
data/lib/masamune/version.rb
CHANGED
data/lib/masamune.rb
CHANGED
@@ -20,8 +20,6 @@
|
|
20
20
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
21
|
# THE SOFTWARE.
|
22
22
|
|
23
|
-
require 'spec_helper'
|
24
|
-
|
25
23
|
describe Masamune::Actions::Transform do
|
26
24
|
let(:environment) { double }
|
27
25
|
let(:catalog) { Masamune::Schema::Catalog.new(environment) }
|
@@ -20,8 +20,6 @@
|
|
20
20
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
21
|
# THE SOFTWARE.
|
22
22
|
|
23
|
-
require 'spec_helper'
|
24
|
-
|
25
23
|
describe Masamune::CachedFilesystem do
|
26
24
|
let(:filesystem) { Masamune::MockFilesystem.new }
|
27
25
|
let(:cached_filesystem) { Masamune::CachedFilesystem.new(filesystem) }
|
@@ -20,8 +20,6 @@
|
|
20
20
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
21
|
# THE SOFTWARE.
|
22
22
|
|
23
|
-
require 'spec_helper'
|
24
|
-
|
25
23
|
describe Masamune::Commands::Hive do
|
26
24
|
let(:filesystem) { Masamune::MockFilesystem.new }
|
27
25
|
let(:configuration) { {:options => options} }
|
@@ -20,8 +20,6 @@
|
|
20
20
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
21
|
# THE SOFTWARE.
|
22
22
|
|
23
|
-
require 'spec_helper'
|
24
|
-
|
25
23
|
describe Masamune::Commands::PostgresAdmin do
|
26
24
|
let(:configuration) { {:create_db_path => 'createdb', :drop_db_path => 'dropdb', :hostname => 'localhost', :username => 'postgres'} }
|
27
25
|
let(:attrs) { {} }
|
@@ -20,8 +20,6 @@
|
|
20
20
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
21
|
# THE SOFTWARE.
|
22
22
|
|
23
|
-
require 'spec_helper'
|
24
|
-
|
25
23
|
describe Masamune::Commands::Postgres do
|
26
24
|
let(:configuration) { {:path => 'psql', :database => 'postgres', :options => options} }
|
27
25
|
let(:options) { [] }
|
@@ -20,8 +20,6 @@
|
|
20
20
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
21
|
# THE SOFTWARE.
|
22
22
|
|
23
|
-
require 'spec_helper'
|
24
|
-
|
25
23
|
describe Masamune::Commands::RetryWithBackoff do
|
26
24
|
let(:options) { {retries: retries, backoff: 0} }
|
27
25
|
let(:delegate) { double }
|
@@ -20,8 +20,6 @@
|
|
20
20
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
21
|
# THE SOFTWARE.
|
22
22
|
|
23
|
-
require 'spec_helper'
|
24
|
-
|
25
23
|
describe Masamune::Configuration do
|
26
24
|
let(:environment) { Masamune::Environment.new }
|
27
25
|
let(:instance) { described_class.new(environment) }
|
@@ -36,6 +34,18 @@ describe Masamune::Configuration do
|
|
36
34
|
it { is_expected.to match(%r{config/masamune\.yml\.erb\Z}) }
|
37
35
|
end
|
38
36
|
|
37
|
+
describe '#as_options' do
|
38
|
+
subject { instance.as_options }
|
39
|
+
it { is_expected.to eq([]) }
|
40
|
+
|
41
|
+
context 'with dry_run: true and debug: true' do
|
42
|
+
before do
|
43
|
+
instance.debug = instance.dry_run = true
|
44
|
+
end
|
45
|
+
it { is_expected.to eq(['--debug', '--dry-run']) }
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
39
49
|
describe '#bind_template' do
|
40
50
|
let(:section) { nil }
|
41
51
|
let(:template) { nil }
|
@@ -20,14 +20,20 @@
|
|
20
20
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
21
|
# THE SOFTWARE.
|
22
22
|
|
23
|
-
require 'spec_helper'
|
24
|
-
|
25
23
|
describe Masamune::DataPlan::Elem do
|
24
|
+
let(:filesystem) { Masamune::MockFilesystem.new }
|
25
|
+
let(:environment) { Masamune::Environment.new }
|
26
26
|
let(:engine) { Masamune::DataPlan::Engine.new }
|
27
|
+
|
28
|
+
before do
|
29
|
+
environment.filesystem = filesystem
|
30
|
+
engine.environment = environment
|
31
|
+
end
|
32
|
+
|
27
33
|
let(:name) { 'primary' }
|
28
34
|
let(:type) { :target }
|
29
|
-
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: 'report/%Y-%m-%d/%H'}) }
|
30
|
-
let(:other_rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: 'log/%Y%m%d.*.log'}) }
|
35
|
+
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: '/report/%Y-%m-%d/%H'}) }
|
36
|
+
let(:other_rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: '/log/%Y%m%d.*.log'}) }
|
31
37
|
|
32
38
|
let(:start_time) { DateTime.civil(2013,07,19,11,07) }
|
33
39
|
let(:other_start_time) { DateTime.civil(2013,07,20,0,0) }
|
@@ -41,7 +47,7 @@ describe Masamune::DataPlan::Elem do
|
|
41
47
|
subject do
|
42
48
|
instance.path
|
43
49
|
end
|
44
|
-
it { is_expected.to eq('report/2013-07-19/11') }
|
50
|
+
it { is_expected.to eq('/report/2013-07-19/11') }
|
45
51
|
end
|
46
52
|
|
47
53
|
describe '#==' do
|
@@ -99,4 +105,66 @@ describe Masamune::DataPlan::Elem do
|
|
99
105
|
it { is_expected.to eq(early) }
|
100
106
|
end
|
101
107
|
end
|
108
|
+
|
109
|
+
describe '#explode' do
|
110
|
+
subject { instance.explode.map(&:path) }
|
111
|
+
|
112
|
+
context 'with free path and existing files' do
|
113
|
+
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: '/report/%Y-%m-%d/%H'}) }
|
114
|
+
before do
|
115
|
+
engine.filesystem.touch!('/report/2013-07-19/11/part-0000')
|
116
|
+
end
|
117
|
+
it { is_expected.to include '/report/2013-07-19/11' }
|
118
|
+
end
|
119
|
+
|
120
|
+
context 'with free path and missing files' do
|
121
|
+
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: '/report/%Y-%m-%d/%H'}) }
|
122
|
+
it { is_expected.to be_empty }
|
123
|
+
end
|
124
|
+
|
125
|
+
context 'with bound path and existing files' do
|
126
|
+
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: '/report/file'}) }
|
127
|
+
before do
|
128
|
+
engine.filesystem.touch!('/report/file')
|
129
|
+
end
|
130
|
+
it { is_expected.to include '/report/file' }
|
131
|
+
end
|
132
|
+
|
133
|
+
context 'with bound path and missing files' do
|
134
|
+
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: '/report/file'}) }
|
135
|
+
it { is_expected.to be_empty }
|
136
|
+
end
|
137
|
+
|
138
|
+
context 'with free table and existing table' do
|
139
|
+
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {table: 'visits_fact', partition: 'y%Y%m'}) }
|
140
|
+
before do
|
141
|
+
expect(instance.rule.engine.postgres_helper).to receive(:table_exists?).and_return(true)
|
142
|
+
end
|
143
|
+
it { is_expected.to include 'visits_fact_y201307' }
|
144
|
+
end
|
145
|
+
|
146
|
+
context 'with free table and missing table' do
|
147
|
+
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {table: 'visits_fact', partition: 'y%Y%m'}) }
|
148
|
+
before do
|
149
|
+
expect(instance.rule.engine.postgres_helper).to receive(:table_exists?).and_return(false)
|
150
|
+
end
|
151
|
+
it { is_expected.to be_empty }
|
152
|
+
end
|
153
|
+
|
154
|
+
context 'with bound table and existing table' do
|
155
|
+
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {table: 'visits_fact'}) }
|
156
|
+
before do
|
157
|
+
expect(instance.rule.engine.postgres_helper).to receive(:table_exists?).and_return(true)
|
158
|
+
end
|
159
|
+
it { is_expected.to include 'visits_fact' }
|
160
|
+
end
|
161
|
+
|
162
|
+
context 'with bound table and missing table' do
|
163
|
+
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {table: 'visits_fact'}) }
|
164
|
+
before do
|
165
|
+
expect(instance.rule.engine.postgres_helper).to receive(:table_exists?).and_return(false)
|
166
|
+
end
|
167
|
+
it { is_expected.to be_empty }
|
168
|
+
end
|
169
|
+
end
|
102
170
|
end
|
@@ -20,8 +20,6 @@
|
|
20
20
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
21
|
# THE SOFTWARE.
|
22
22
|
|
23
|
-
require 'spec_helper'
|
24
|
-
|
25
23
|
describe Masamune::DataPlan::Engine do
|
26
24
|
let(:filesystem) { Masamune::MockFilesystem.new }
|
27
25
|
let(:environment) { Masamune::Environment.new }
|
@@ -20,8 +20,6 @@
|
|
20
20
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
21
|
# THE SOFTWARE.
|
22
22
|
|
23
|
-
require 'spec_helper'
|
24
|
-
|
25
23
|
describe Masamune::DataPlan::Rule do
|
26
24
|
let(:engine) { Masamune::DataPlan::Engine.new }
|
27
25
|
let(:name) { 'primary' }
|
@@ -138,6 +136,27 @@ describe Masamune::DataPlan::Rule do
|
|
138
136
|
let(:input) { instance.bind_input(prev_input) }
|
139
137
|
it { is_expected.to eq(input) }
|
140
138
|
end
|
139
|
+
|
140
|
+
context 'with wildcard pattern' do
|
141
|
+
let(:pattern) { 'requests/y=%Y/m=%-m/d=%-d/h=%-k/*' }
|
142
|
+
let(:input) { 'requests/y=2013/m=4/d=30/h=20/part-00000' }
|
143
|
+
let(:output_date) { DateTime.civil(2013,04,30,20) }
|
144
|
+
|
145
|
+
describe '#path' do
|
146
|
+
subject { elem.path }
|
147
|
+
it { is_expected.to eq(input) }
|
148
|
+
end
|
149
|
+
|
150
|
+
describe '#start_time' do
|
151
|
+
subject { elem.start_time }
|
152
|
+
it { is_expected.to eq(output_date) }
|
153
|
+
end
|
154
|
+
|
155
|
+
describe '#stop_time' do
|
156
|
+
subject { elem.stop_time }
|
157
|
+
it { is_expected.to eq(output_date.to_time + 1.hour) }
|
158
|
+
end
|
159
|
+
end
|
141
160
|
end
|
142
161
|
|
143
162
|
describe '#unify' do
|
@@ -200,7 +219,7 @@ describe Masamune::DataPlan::Rule do
|
|
200
219
|
it { is_expected.to eq(true) }
|
201
220
|
end
|
202
221
|
|
203
|
-
context 'with alternative hour' do
|
222
|
+
context 'with another alternative hour' do
|
204
223
|
let(:pattern) { 'requests/y=%Y/m=%-m/d=%-d/h=%-k' }
|
205
224
|
let(:input) { 'requests/y=2013/m=4/d=30/h=20' }
|
206
225
|
it { is_expected.to eq(true) }
|
@@ -212,9 +231,9 @@ describe Masamune::DataPlan::Rule do
|
|
212
231
|
it { is_expected.to eq(true) }
|
213
232
|
end
|
214
233
|
|
215
|
-
context 'with
|
216
|
-
let(:pattern) { '
|
217
|
-
let(:input) { '
|
234
|
+
context 'with wildcard input' do
|
235
|
+
let(:pattern) { 'requests/y=%Y/m=%-m/d=%-d/h=%-k' }
|
236
|
+
let(:input) { 'requests/y=2013/m=4/d=30/h=20/*' }
|
218
237
|
it { is_expected.to eq(true) }
|
219
238
|
end
|
220
239
|
end
|
@@ -419,4 +438,30 @@ describe Masamune::DataPlan::Rule do
|
|
419
438
|
end
|
420
439
|
end
|
421
440
|
end
|
441
|
+
|
442
|
+
describe '#free? ' do
|
443
|
+
subject { instance.free? }
|
444
|
+
context 'with rule that contains free variables' do
|
445
|
+
let(:pattern) { 'report/%Y-%m-%d/%H' }
|
446
|
+
it { is_expected.to be(true) }
|
447
|
+
end
|
448
|
+
|
449
|
+
context 'with rule that does not contain free variables' do
|
450
|
+
let(:pattern) { 'report/file' }
|
451
|
+
it { is_expected.to be(false) }
|
452
|
+
end
|
453
|
+
end
|
454
|
+
|
455
|
+
describe '#bound? ' do
|
456
|
+
subject { instance.bound? }
|
457
|
+
context 'with rule that contains free variables' do
|
458
|
+
let(:pattern) { 'report/%Y-%m-%d/%H' }
|
459
|
+
it { is_expected.to be(false) }
|
460
|
+
end
|
461
|
+
|
462
|
+
context 'with rule that does not contain free variables' do
|
463
|
+
let(:pattern) { 'report/file' }
|
464
|
+
it { is_expected.to be(true) }
|
465
|
+
end
|
466
|
+
end
|
422
467
|
end
|
@@ -20,8 +20,6 @@
|
|
20
20
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
21
|
# THE SOFTWARE.
|
22
22
|
|
23
|
-
require 'spec_helper'
|
24
|
-
|
25
23
|
describe Masamune::DataPlan::Set do
|
26
24
|
let(:fs) { Masamune::MockFilesystem.new }
|
27
25
|
let!(:engine) { Masamune::DataPlan::Engine.new }
|
@@ -243,13 +241,12 @@ describe Masamune::DataPlan::Set do
|
|
243
241
|
it { expect(stale_targets).to include '/table/y=2013/m=01/d=01' }
|
244
242
|
end
|
245
243
|
|
246
|
-
context 'when
|
244
|
+
context 'when none stale targets (tie breaker)' do
|
247
245
|
before do
|
248
246
|
fs.touch!('/log/20130101.random_1.log', mtime: present_time)
|
249
247
|
end
|
250
248
|
|
251
|
-
it { expect(stale_targets
|
252
|
-
it { expect(stale_targets).to include '/table/y=2013/m=01/d=01' }
|
249
|
+
it { expect(stale_targets).to be_empty }
|
253
250
|
end
|
254
251
|
|
255
252
|
context 'when all stale targets' do
|
@@ -20,8 +20,6 @@
|
|
20
20
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
21
|
# THE SOFTWARE.
|
22
22
|
|
23
|
-
require 'spec_helper'
|
24
|
-
|
25
23
|
require 'securerandom'
|
26
24
|
|
27
25
|
# NOTE when operating between hdfs and s3, hadoop fs requires s3n URI
|
@@ -62,6 +60,10 @@ shared_examples_for 'Filesystem' do
|
|
62
60
|
it { expect(instance.get_path(:home_dir, '/a/b', 'c')).to eq('/home/a/b/c') }
|
63
61
|
end
|
64
62
|
|
63
|
+
context 'with extra options' do
|
64
|
+
it { expect(instance.get_path(:home_dir, '/a/b', 'c', mkdir: true)).to eq('/home/a/b/c') }
|
65
|
+
end
|
66
|
+
|
65
67
|
context 'with parameter substitution' do
|
66
68
|
before do
|
67
69
|
instance.configuration.params[:user] = 'zombo'
|
@@ -373,6 +375,14 @@ shared_examples_for 'Filesystem' do
|
|
373
375
|
it { is_expected.to eq(true) }
|
374
376
|
end
|
375
377
|
|
378
|
+
context 'local missing file after glob' do
|
379
|
+
before do
|
380
|
+
instance.glob(new_file + '/*')
|
381
|
+
end
|
382
|
+
subject { instance.exists?(new_file) }
|
383
|
+
it { is_expected.to eq(false) }
|
384
|
+
end
|
385
|
+
|
376
386
|
context 'hdfs existing file' do
|
377
387
|
before do
|
378
388
|
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.dirname(old_dir) + '/*', safe: true).at_most(:once).
|
@@ -567,6 +577,15 @@ shared_examples_for 'Filesystem' do
|
|
567
577
|
it { is_expected.to eq(true) }
|
568
578
|
end
|
569
579
|
|
580
|
+
context 'local existing directory' do
|
581
|
+
subject { Dir.exists?(old_dir) }
|
582
|
+
before do
|
583
|
+
expect(FileUtils).to receive(:mkdir_p).never
|
584
|
+
instance.mkdir!(old_dir)
|
585
|
+
end
|
586
|
+
it { is_expected.to eq(true) }
|
587
|
+
end
|
588
|
+
|
570
589
|
context 'hdfs directory' do
|
571
590
|
it do
|
572
591
|
expect(filesystem).to receive(:hadoop_fs).with('-mkdir', '-p', 'file://' + new_dir, 'file://' + other_new_dir).once
|
@@ -610,8 +629,18 @@ shared_examples_for 'Filesystem' do
|
|
610
629
|
it { expect { |b| instance.glob(pattern, &b) }.to yield_successive_args(old_dir, old_file) }
|
611
630
|
end
|
612
631
|
|
613
|
-
context 'local one matches
|
614
|
-
let(:pattern) { File.join(File.dirname(
|
632
|
+
context 'local one matches with glob' do
|
633
|
+
let(:pattern) { File.join(File.dirname(old_dir), '*') }
|
634
|
+
it 'has 1 item' do
|
635
|
+
expect(subject.count).to eq(2)
|
636
|
+
end
|
637
|
+
it { is_expected.to include old_dir }
|
638
|
+
it { is_expected.to include old_file }
|
639
|
+
it { expect { |b| instance.glob(pattern, &b) }.to yield_successive_args(old_dir, old_file) }
|
640
|
+
end
|
641
|
+
|
642
|
+
context 'local one matches with glob and suffix' do
|
643
|
+
let(:pattern) { File.join(File.dirname(old_dir), '*.txt') }
|
615
644
|
it 'has 1 item' do
|
616
645
|
expect(subject.count).to eq(1)
|
617
646
|
end
|