masamune 0.14.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +10 -1
- data/Rakefile +37 -0
- data/bin/masamune-dump +22 -0
- data/bin/masamune-elastic-mapreduce +22 -0
- data/bin/masamune-hive +22 -0
- data/bin/masamune-psql +22 -0
- data/bin/masamune-shell +22 -0
- data/lib/masamune/cached_filesystem.rb +1 -1
- data/lib/masamune/commands/shell.rb +1 -1
- data/lib/masamune/configuration.rb +5 -7
- data/lib/masamune/data_plan/elem.rb +15 -7
- data/lib/masamune/data_plan/engine.rb +2 -0
- data/lib/masamune/data_plan/rule.rb +16 -5
- data/lib/masamune/data_plan/set.rb +8 -8
- data/lib/masamune/filesystem.rb +12 -6
- data/lib/masamune/schema/catalog.rb +6 -6
- data/lib/masamune/schema/column.rb +1 -1
- data/lib/masamune/schema/map.rb +6 -2
- data/lib/masamune/schema/store.rb +31 -3
- data/lib/masamune/tasks/shell_thor.rb +1 -1
- data/lib/masamune/thor.rb +12 -4
- data/lib/masamune/version.rb +1 -1
- data/lib/masamune.rb +0 -1
- data/spec/masamune/actions/elastic_mapreduce_spec.rb +0 -2
- data/spec/masamune/actions/execute_spec.rb +0 -2
- data/spec/masamune/actions/hadoop_filesystem_spec.rb +0 -2
- data/spec/masamune/actions/hadoop_streaming_spec.rb +0 -2
- data/spec/masamune/actions/hive_spec.rb +0 -2
- data/spec/masamune/actions/invoke_parallel_spec.rb +0 -2
- data/spec/masamune/actions/postgres_admin_spec.rb +0 -2
- data/spec/masamune/actions/postgres_spec.rb +0 -2
- data/spec/masamune/actions/s3cmd_spec.rb +0 -2
- data/spec/masamune/actions/transform_spec.rb +0 -2
- data/spec/masamune/after_initialization_callbacks_spec.rb +0 -2
- data/spec/masamune/cached_filesystem_spec.rb +0 -2
- data/spec/masamune/commands/hadoop_filesystem_spec.rb +0 -2
- data/spec/masamune/commands/hadoop_streaming_spec.rb +0 -2
- data/spec/masamune/commands/hive_spec.rb +0 -2
- data/spec/masamune/commands/postgres_admin_spec.rb +0 -2
- data/spec/masamune/commands/postgres_spec.rb +0 -2
- data/spec/masamune/commands/retry_with_backoff_spec.rb +0 -2
- data/spec/masamune/commands/s3cmd_spec.rb +0 -2
- data/spec/masamune/commands/shell_spec.rb +0 -2
- data/spec/masamune/configuration_spec.rb +12 -2
- data/spec/masamune/data_plan/builder_spec.rb +0 -2
- data/spec/masamune/data_plan/elem_spec.rb +73 -5
- data/spec/masamune/data_plan/engine_spec.rb +0 -2
- data/spec/masamune/data_plan/rule_spec.rb +51 -6
- data/spec/masamune/data_plan/set_spec.rb +2 -5
- data/spec/masamune/environment_spec.rb +0 -2
- data/spec/masamune/filesystem_spec.rb +33 -4
- data/spec/masamune/helpers/postgres_spec.rb +0 -2
- data/spec/masamune/rspec/job_fixture_spec.rb +365 -0
- data/spec/masamune/rspec/shared_example_group_spec.rb +73 -0
- data/spec/masamune/schema/catalog_spec.rb +14 -2
- data/spec/masamune/schema/column_spec.rb +0 -2
- data/spec/masamune/schema/dimension_spec.rb +0 -2
- data/spec/masamune/schema/fact_spec.rb +0 -2
- data/spec/masamune/schema/map_spec.rb +51 -2
- data/spec/masamune/schema/row_spec.rb +0 -2
- data/spec/masamune/schema/store_spec.rb +23 -2
- data/spec/masamune/schema/table_spec.rb +0 -2
- data/spec/masamune/string_format_spec.rb +0 -2
- data/spec/masamune/tasks/dump_thor_spec.rb +0 -3
- data/spec/masamune/tasks/elastic_mapreduce_thor_spec.rb +0 -3
- data/spec/masamune/tasks/hive_thor_spec.rb +0 -3
- data/spec/masamune/tasks/postgres_thor_spec.rb +0 -3
- data/spec/masamune/tasks/shell_thor_spec.rb +0 -3
- data/spec/masamune/template_spec.rb +0 -2
- data/spec/masamune/thor_spec.rb +53 -8
- data/spec/masamune/transform/bulk_upsert.dimension_spec.rb +0 -2
- data/spec/masamune/transform/consolidate_dimension_spec.rb +0 -2
- data/spec/masamune/transform/deduplicate_dimension_spec.rb +0 -2
- data/spec/masamune/transform/define_schema_spec.rb +0 -2
- data/spec/masamune/transform/define_table.dimension_spec.rb +0 -2
- data/spec/masamune/transform/define_table.fact_spec.rb +0 -2
- data/spec/masamune/transform/define_table.table_spec.rb +0 -2
- data/spec/masamune/transform/denormalize_table_spec.rb +0 -2
- data/spec/masamune/transform/insert_reference_values.dimension_spec.rb +0 -2
- data/spec/masamune/transform/insert_reference_values.fact_spec.rb +0 -2
- data/spec/masamune/transform/load_dimension_spec.rb +0 -2
- data/spec/masamune/transform/load_fact_spec.rb +0 -2
- data/spec/masamune/transform/relabel_dimension_spec.rb +0 -2
- data/spec/masamune/transform/rollup_fact_spec.rb +0 -2
- data/spec/masamune/transform/snapshot_dimension_spec.rb +0 -2
- data/spec/masamune/transform/stage_dimension_spec.rb +0 -2
- data/spec/masamune/transform/stage_fact_spec.rb +0 -2
- data/spec/masamune_spec.rb +0 -2
- data/spec/spec_helper.rb +2 -0
- data/spec/support/masamune/job_example_group.rb +62 -0
- data/spec/support/masamune/job_fixture.rb +137 -0
- data/spec/support/masamune/shared_example_group.rb +203 -0
- data/spec/support/masamune/step_example_group.rb +68 -0
- data/spec/support/masamune/step_fixture.rb +91 -0
- data/{lib/masamune/thor_loader.rb → spec/support/masamune/task_example_group.rb} +33 -10
- data/spec/support/rspec/example/action_example_group.rb +1 -1
- metadata +32 -3
data/lib/masamune/thor.rb
CHANGED
@@ -72,11 +72,14 @@ module Masamune
|
|
72
72
|
raise e unless %w(SIGHUP SIGTERM).include?(e.to_s)
|
73
73
|
instance.logger.debug("Exiting at user request on #{e.to_s}")
|
74
74
|
exit 0
|
75
|
+
rescue ::Thor::MalformattedArgumentError, ::Thor::RequiredArgumentMissingError => e
|
76
|
+
raise e
|
75
77
|
rescue => e
|
76
78
|
instance.logger.error("#{e.message} (#{e.class}) backtrace:")
|
77
79
|
e.backtrace.each { |x| instance.logger.error(x) }
|
80
|
+
$stderr.puts e.to_s
|
78
81
|
$stderr.puts "For complete debug log see: #{instance.log_file_name.to_s}"
|
79
|
-
|
82
|
+
exit 1
|
80
83
|
end
|
81
84
|
end
|
82
85
|
|
@@ -96,8 +99,7 @@ module Masamune
|
|
96
99
|
class_option :quiet, :type => :boolean, :aliases => '-q', :desc => 'Suppress all output', :default => false
|
97
100
|
class_option :verbose, :type => :boolean, :aliases => '-v', :desc => 'Print command execution information', :default => false
|
98
101
|
class_option :debug, :type => :boolean, :aliases => '-d', :desc => 'Print debugging information', :default => false
|
99
|
-
class_option :
|
100
|
-
class_option :dry_run, :type => :boolean, :aliases => '-n', :desc => 'Combination of --no-op and --verbose', :default => false
|
102
|
+
class_option :dry_run, :type => :boolean, :aliases => '-n', :desc => 'Do not execute commands that modify state', :default => false
|
101
103
|
class_option :config, :desc => 'Configuration file'
|
102
104
|
class_option :version, :desc => 'Print version and exit', :type => :boolean
|
103
105
|
class_option :lock, :desc => 'Optional job lock name', :type => :string
|
@@ -111,6 +113,8 @@ module Masamune
|
|
111
113
|
self.current_command_name = current_namespace ? current_namespace + ':' + current_task_name : current_task_name
|
112
114
|
self.class.instance = self
|
113
115
|
|
116
|
+
define_current_dir
|
117
|
+
|
114
118
|
if _options.is_a?(Array)
|
115
119
|
_options, self.extra = self.class.parse_extra(_options)
|
116
120
|
end
|
@@ -137,7 +141,6 @@ module Masamune
|
|
137
141
|
config.quiet = options[:quiet]
|
138
142
|
config.verbose = options[:verbose] || options[:dry_run]
|
139
143
|
config.debug = options[:debug]
|
140
|
-
config.no_op = options[:no_op] || options[:dry_run]
|
141
144
|
config.dry_run = options[:dry_run]
|
142
145
|
config.lock = options[:lock]
|
143
146
|
|
@@ -195,6 +198,11 @@ module Masamune
|
|
195
198
|
|
196
199
|
private
|
197
200
|
|
201
|
+
def define_current_dir
|
202
|
+
return unless current_task_name
|
203
|
+
filesystem.add_path(:current_dir, File.dirname(method(current_task_name).source_location.first))
|
204
|
+
end
|
205
|
+
|
198
206
|
def display_help?
|
199
207
|
options[:help] || current_task_name == 'help'
|
200
208
|
end
|
data/lib/masamune/version.rb
CHANGED
data/lib/masamune.rb
CHANGED
@@ -20,8 +20,6 @@
|
|
20
20
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
21
|
# THE SOFTWARE.
|
22
22
|
|
23
|
-
require 'spec_helper'
|
24
|
-
|
25
23
|
describe Masamune::Actions::Transform do
|
26
24
|
let(:environment) { double }
|
27
25
|
let(:catalog) { Masamune::Schema::Catalog.new(environment) }
|
@@ -20,8 +20,6 @@
|
|
20
20
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
21
|
# THE SOFTWARE.
|
22
22
|
|
23
|
-
require 'spec_helper'
|
24
|
-
|
25
23
|
describe Masamune::CachedFilesystem do
|
26
24
|
let(:filesystem) { Masamune::MockFilesystem.new }
|
27
25
|
let(:cached_filesystem) { Masamune::CachedFilesystem.new(filesystem) }
|
@@ -20,8 +20,6 @@
|
|
20
20
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
21
|
# THE SOFTWARE.
|
22
22
|
|
23
|
-
require 'spec_helper'
|
24
|
-
|
25
23
|
describe Masamune::Commands::Hive do
|
26
24
|
let(:filesystem) { Masamune::MockFilesystem.new }
|
27
25
|
let(:configuration) { {:options => options} }
|
@@ -20,8 +20,6 @@
|
|
20
20
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
21
|
# THE SOFTWARE.
|
22
22
|
|
23
|
-
require 'spec_helper'
|
24
|
-
|
25
23
|
describe Masamune::Commands::PostgresAdmin do
|
26
24
|
let(:configuration) { {:create_db_path => 'createdb', :drop_db_path => 'dropdb', :hostname => 'localhost', :username => 'postgres'} }
|
27
25
|
let(:attrs) { {} }
|
@@ -20,8 +20,6 @@
|
|
20
20
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
21
|
# THE SOFTWARE.
|
22
22
|
|
23
|
-
require 'spec_helper'
|
24
|
-
|
25
23
|
describe Masamune::Commands::Postgres do
|
26
24
|
let(:configuration) { {:path => 'psql', :database => 'postgres', :options => options} }
|
27
25
|
let(:options) { [] }
|
@@ -20,8 +20,6 @@
|
|
20
20
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
21
|
# THE SOFTWARE.
|
22
22
|
|
23
|
-
require 'spec_helper'
|
24
|
-
|
25
23
|
describe Masamune::Commands::RetryWithBackoff do
|
26
24
|
let(:options) { {retries: retries, backoff: 0} }
|
27
25
|
let(:delegate) { double }
|
@@ -20,8 +20,6 @@
|
|
20
20
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
21
|
# THE SOFTWARE.
|
22
22
|
|
23
|
-
require 'spec_helper'
|
24
|
-
|
25
23
|
describe Masamune::Configuration do
|
26
24
|
let(:environment) { Masamune::Environment.new }
|
27
25
|
let(:instance) { described_class.new(environment) }
|
@@ -36,6 +34,18 @@ describe Masamune::Configuration do
|
|
36
34
|
it { is_expected.to match(%r{config/masamune\.yml\.erb\Z}) }
|
37
35
|
end
|
38
36
|
|
37
|
+
describe '#as_options' do
|
38
|
+
subject { instance.as_options }
|
39
|
+
it { is_expected.to eq([]) }
|
40
|
+
|
41
|
+
context 'with dry_run: true and debug: true' do
|
42
|
+
before do
|
43
|
+
instance.debug = instance.dry_run = true
|
44
|
+
end
|
45
|
+
it { is_expected.to eq(['--debug', '--dry-run']) }
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
39
49
|
describe '#bind_template' do
|
40
50
|
let(:section) { nil }
|
41
51
|
let(:template) { nil }
|
@@ -20,14 +20,20 @@
|
|
20
20
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
21
|
# THE SOFTWARE.
|
22
22
|
|
23
|
-
require 'spec_helper'
|
24
|
-
|
25
23
|
describe Masamune::DataPlan::Elem do
|
24
|
+
let(:filesystem) { Masamune::MockFilesystem.new }
|
25
|
+
let(:environment) { Masamune::Environment.new }
|
26
26
|
let(:engine) { Masamune::DataPlan::Engine.new }
|
27
|
+
|
28
|
+
before do
|
29
|
+
environment.filesystem = filesystem
|
30
|
+
engine.environment = environment
|
31
|
+
end
|
32
|
+
|
27
33
|
let(:name) { 'primary' }
|
28
34
|
let(:type) { :target }
|
29
|
-
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: 'report/%Y-%m-%d/%H'}) }
|
30
|
-
let(:other_rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: 'log/%Y%m%d.*.log'}) }
|
35
|
+
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: '/report/%Y-%m-%d/%H'}) }
|
36
|
+
let(:other_rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: '/log/%Y%m%d.*.log'}) }
|
31
37
|
|
32
38
|
let(:start_time) { DateTime.civil(2013,07,19,11,07) }
|
33
39
|
let(:other_start_time) { DateTime.civil(2013,07,20,0,0) }
|
@@ -41,7 +47,7 @@ describe Masamune::DataPlan::Elem do
|
|
41
47
|
subject do
|
42
48
|
instance.path
|
43
49
|
end
|
44
|
-
it { is_expected.to eq('report/2013-07-19/11') }
|
50
|
+
it { is_expected.to eq('/report/2013-07-19/11') }
|
45
51
|
end
|
46
52
|
|
47
53
|
describe '#==' do
|
@@ -99,4 +105,66 @@ describe Masamune::DataPlan::Elem do
|
|
99
105
|
it { is_expected.to eq(early) }
|
100
106
|
end
|
101
107
|
end
|
108
|
+
|
109
|
+
describe '#explode' do
|
110
|
+
subject { instance.explode.map(&:path) }
|
111
|
+
|
112
|
+
context 'with free path and existing files' do
|
113
|
+
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: '/report/%Y-%m-%d/%H'}) }
|
114
|
+
before do
|
115
|
+
engine.filesystem.touch!('/report/2013-07-19/11/part-0000')
|
116
|
+
end
|
117
|
+
it { is_expected.to include '/report/2013-07-19/11' }
|
118
|
+
end
|
119
|
+
|
120
|
+
context 'with free path and missing files' do
|
121
|
+
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: '/report/%Y-%m-%d/%H'}) }
|
122
|
+
it { is_expected.to be_empty }
|
123
|
+
end
|
124
|
+
|
125
|
+
context 'with bound path and existing files' do
|
126
|
+
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: '/report/file'}) }
|
127
|
+
before do
|
128
|
+
engine.filesystem.touch!('/report/file')
|
129
|
+
end
|
130
|
+
it { is_expected.to include '/report/file' }
|
131
|
+
end
|
132
|
+
|
133
|
+
context 'with bound path and missing files' do
|
134
|
+
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: '/report/file'}) }
|
135
|
+
it { is_expected.to be_empty }
|
136
|
+
end
|
137
|
+
|
138
|
+
context 'with free table and existing table' do
|
139
|
+
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {table: 'visits_fact', partition: 'y%Y%m'}) }
|
140
|
+
before do
|
141
|
+
expect(instance.rule.engine.postgres_helper).to receive(:table_exists?).and_return(true)
|
142
|
+
end
|
143
|
+
it { is_expected.to include 'visits_fact_y201307' }
|
144
|
+
end
|
145
|
+
|
146
|
+
context 'with free table and missing table' do
|
147
|
+
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {table: 'visits_fact', partition: 'y%Y%m'}) }
|
148
|
+
before do
|
149
|
+
expect(instance.rule.engine.postgres_helper).to receive(:table_exists?).and_return(false)
|
150
|
+
end
|
151
|
+
it { is_expected.to be_empty }
|
152
|
+
end
|
153
|
+
|
154
|
+
context 'with bound table and existing table' do
|
155
|
+
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {table: 'visits_fact'}) }
|
156
|
+
before do
|
157
|
+
expect(instance.rule.engine.postgres_helper).to receive(:table_exists?).and_return(true)
|
158
|
+
end
|
159
|
+
it { is_expected.to include 'visits_fact' }
|
160
|
+
end
|
161
|
+
|
162
|
+
context 'with bound table and missing table' do
|
163
|
+
let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {table: 'visits_fact'}) }
|
164
|
+
before do
|
165
|
+
expect(instance.rule.engine.postgres_helper).to receive(:table_exists?).and_return(false)
|
166
|
+
end
|
167
|
+
it { is_expected.to be_empty }
|
168
|
+
end
|
169
|
+
end
|
102
170
|
end
|
@@ -20,8 +20,6 @@
|
|
20
20
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
21
|
# THE SOFTWARE.
|
22
22
|
|
23
|
-
require 'spec_helper'
|
24
|
-
|
25
23
|
describe Masamune::DataPlan::Engine do
|
26
24
|
let(:filesystem) { Masamune::MockFilesystem.new }
|
27
25
|
let(:environment) { Masamune::Environment.new }
|
@@ -20,8 +20,6 @@
|
|
20
20
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
21
|
# THE SOFTWARE.
|
22
22
|
|
23
|
-
require 'spec_helper'
|
24
|
-
|
25
23
|
describe Masamune::DataPlan::Rule do
|
26
24
|
let(:engine) { Masamune::DataPlan::Engine.new }
|
27
25
|
let(:name) { 'primary' }
|
@@ -138,6 +136,27 @@ describe Masamune::DataPlan::Rule do
|
|
138
136
|
let(:input) { instance.bind_input(prev_input) }
|
139
137
|
it { is_expected.to eq(input) }
|
140
138
|
end
|
139
|
+
|
140
|
+
context 'with wildcard pattern' do
|
141
|
+
let(:pattern) { 'requests/y=%Y/m=%-m/d=%-d/h=%-k/*' }
|
142
|
+
let(:input) { 'requests/y=2013/m=4/d=30/h=20/part-00000' }
|
143
|
+
let(:output_date) { DateTime.civil(2013,04,30,20) }
|
144
|
+
|
145
|
+
describe '#path' do
|
146
|
+
subject { elem.path }
|
147
|
+
it { is_expected.to eq(input) }
|
148
|
+
end
|
149
|
+
|
150
|
+
describe '#start_time' do
|
151
|
+
subject { elem.start_time }
|
152
|
+
it { is_expected.to eq(output_date) }
|
153
|
+
end
|
154
|
+
|
155
|
+
describe '#stop_time' do
|
156
|
+
subject { elem.stop_time }
|
157
|
+
it { is_expected.to eq(output_date.to_time + 1.hour) }
|
158
|
+
end
|
159
|
+
end
|
141
160
|
end
|
142
161
|
|
143
162
|
describe '#unify' do
|
@@ -200,7 +219,7 @@ describe Masamune::DataPlan::Rule do
|
|
200
219
|
it { is_expected.to eq(true) }
|
201
220
|
end
|
202
221
|
|
203
|
-
context 'with alternative hour' do
|
222
|
+
context 'with another alternative hour' do
|
204
223
|
let(:pattern) { 'requests/y=%Y/m=%-m/d=%-d/h=%-k' }
|
205
224
|
let(:input) { 'requests/y=2013/m=4/d=30/h=20' }
|
206
225
|
it { is_expected.to eq(true) }
|
@@ -212,9 +231,9 @@ describe Masamune::DataPlan::Rule do
|
|
212
231
|
it { is_expected.to eq(true) }
|
213
232
|
end
|
214
233
|
|
215
|
-
context 'with
|
216
|
-
let(:pattern) { '
|
217
|
-
let(:input) { '
|
234
|
+
context 'with wildcard input' do
|
235
|
+
let(:pattern) { 'requests/y=%Y/m=%-m/d=%-d/h=%-k' }
|
236
|
+
let(:input) { 'requests/y=2013/m=4/d=30/h=20/*' }
|
218
237
|
it { is_expected.to eq(true) }
|
219
238
|
end
|
220
239
|
end
|
@@ -419,4 +438,30 @@ describe Masamune::DataPlan::Rule do
|
|
419
438
|
end
|
420
439
|
end
|
421
440
|
end
|
441
|
+
|
442
|
+
describe '#free? ' do
|
443
|
+
subject { instance.free? }
|
444
|
+
context 'with rule that contains free variables' do
|
445
|
+
let(:pattern) { 'report/%Y-%m-%d/%H' }
|
446
|
+
it { is_expected.to be(true) }
|
447
|
+
end
|
448
|
+
|
449
|
+
context 'with rule that does not contain free variables' do
|
450
|
+
let(:pattern) { 'report/file' }
|
451
|
+
it { is_expected.to be(false) }
|
452
|
+
end
|
453
|
+
end
|
454
|
+
|
455
|
+
describe '#bound? ' do
|
456
|
+
subject { instance.bound? }
|
457
|
+
context 'with rule that contains free variables' do
|
458
|
+
let(:pattern) { 'report/%Y-%m-%d/%H' }
|
459
|
+
it { is_expected.to be(false) }
|
460
|
+
end
|
461
|
+
|
462
|
+
context 'with rule that does not contain free variables' do
|
463
|
+
let(:pattern) { 'report/file' }
|
464
|
+
it { is_expected.to be(true) }
|
465
|
+
end
|
466
|
+
end
|
422
467
|
end
|
@@ -20,8 +20,6 @@
|
|
20
20
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
21
|
# THE SOFTWARE.
|
22
22
|
|
23
|
-
require 'spec_helper'
|
24
|
-
|
25
23
|
describe Masamune::DataPlan::Set do
|
26
24
|
let(:fs) { Masamune::MockFilesystem.new }
|
27
25
|
let!(:engine) { Masamune::DataPlan::Engine.new }
|
@@ -243,13 +241,12 @@ describe Masamune::DataPlan::Set do
|
|
243
241
|
it { expect(stale_targets).to include '/table/y=2013/m=01/d=01' }
|
244
242
|
end
|
245
243
|
|
246
|
-
context 'when
|
244
|
+
context 'when none stale targets (tie breaker)' do
|
247
245
|
before do
|
248
246
|
fs.touch!('/log/20130101.random_1.log', mtime: present_time)
|
249
247
|
end
|
250
248
|
|
251
|
-
it { expect(stale_targets
|
252
|
-
it { expect(stale_targets).to include '/table/y=2013/m=01/d=01' }
|
249
|
+
it { expect(stale_targets).to be_empty }
|
253
250
|
end
|
254
251
|
|
255
252
|
context 'when all stale targets' do
|
@@ -20,8 +20,6 @@
|
|
20
20
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
21
|
# THE SOFTWARE.
|
22
22
|
|
23
|
-
require 'spec_helper'
|
24
|
-
|
25
23
|
require 'securerandom'
|
26
24
|
|
27
25
|
# NOTE when operating between hdfs and s3, hadoop fs requires s3n URI
|
@@ -62,6 +60,10 @@ shared_examples_for 'Filesystem' do
|
|
62
60
|
it { expect(instance.get_path(:home_dir, '/a/b', 'c')).to eq('/home/a/b/c') }
|
63
61
|
end
|
64
62
|
|
63
|
+
context 'with extra options' do
|
64
|
+
it { expect(instance.get_path(:home_dir, '/a/b', 'c', mkdir: true)).to eq('/home/a/b/c') }
|
65
|
+
end
|
66
|
+
|
65
67
|
context 'with parameter substitution' do
|
66
68
|
before do
|
67
69
|
instance.configuration.params[:user] = 'zombo'
|
@@ -373,6 +375,14 @@ shared_examples_for 'Filesystem' do
|
|
373
375
|
it { is_expected.to eq(true) }
|
374
376
|
end
|
375
377
|
|
378
|
+
context 'local missing file after glob' do
|
379
|
+
before do
|
380
|
+
instance.glob(new_file + '/*')
|
381
|
+
end
|
382
|
+
subject { instance.exists?(new_file) }
|
383
|
+
it { is_expected.to eq(false) }
|
384
|
+
end
|
385
|
+
|
376
386
|
context 'hdfs existing file' do
|
377
387
|
before do
|
378
388
|
expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.dirname(old_dir) + '/*', safe: true).at_most(:once).
|
@@ -567,6 +577,15 @@ shared_examples_for 'Filesystem' do
|
|
567
577
|
it { is_expected.to eq(true) }
|
568
578
|
end
|
569
579
|
|
580
|
+
context 'local existing directory' do
|
581
|
+
subject { Dir.exists?(old_dir) }
|
582
|
+
before do
|
583
|
+
expect(FileUtils).to receive(:mkdir_p).never
|
584
|
+
instance.mkdir!(old_dir)
|
585
|
+
end
|
586
|
+
it { is_expected.to eq(true) }
|
587
|
+
end
|
588
|
+
|
570
589
|
context 'hdfs directory' do
|
571
590
|
it do
|
572
591
|
expect(filesystem).to receive(:hadoop_fs).with('-mkdir', '-p', 'file://' + new_dir, 'file://' + other_new_dir).once
|
@@ -610,8 +629,18 @@ shared_examples_for 'Filesystem' do
|
|
610
629
|
it { expect { |b| instance.glob(pattern, &b) }.to yield_successive_args(old_dir, old_file) }
|
611
630
|
end
|
612
631
|
|
613
|
-
context 'local one matches
|
614
|
-
let(:pattern) { File.join(File.dirname(
|
632
|
+
context 'local one matches with glob' do
|
633
|
+
let(:pattern) { File.join(File.dirname(old_dir), '*') }
|
634
|
+
it 'has 1 item' do
|
635
|
+
expect(subject.count).to eq(2)
|
636
|
+
end
|
637
|
+
it { is_expected.to include old_dir }
|
638
|
+
it { is_expected.to include old_file }
|
639
|
+
it { expect { |b| instance.glob(pattern, &b) }.to yield_successive_args(old_dir, old_file) }
|
640
|
+
end
|
641
|
+
|
642
|
+
context 'local one matches with glob and suffix' do
|
643
|
+
let(:pattern) { File.join(File.dirname(old_dir), '*.txt') }
|
615
644
|
it 'has 1 item' do
|
616
645
|
expect(subject.count).to eq(1)
|
617
646
|
end
|