masamune 0.14.0 → 0.15.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (98) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +10 -1
  3. data/Rakefile +37 -0
  4. data/bin/masamune-dump +22 -0
  5. data/bin/masamune-elastic-mapreduce +22 -0
  6. data/bin/masamune-hive +22 -0
  7. data/bin/masamune-psql +22 -0
  8. data/bin/masamune-shell +22 -0
  9. data/lib/masamune/cached_filesystem.rb +1 -1
  10. data/lib/masamune/commands/shell.rb +1 -1
  11. data/lib/masamune/configuration.rb +5 -7
  12. data/lib/masamune/data_plan/elem.rb +15 -7
  13. data/lib/masamune/data_plan/engine.rb +2 -0
  14. data/lib/masamune/data_plan/rule.rb +16 -5
  15. data/lib/masamune/data_plan/set.rb +8 -8
  16. data/lib/masamune/filesystem.rb +12 -6
  17. data/lib/masamune/schema/catalog.rb +6 -6
  18. data/lib/masamune/schema/column.rb +1 -1
  19. data/lib/masamune/schema/map.rb +6 -2
  20. data/lib/masamune/schema/store.rb +31 -3
  21. data/lib/masamune/tasks/shell_thor.rb +1 -1
  22. data/lib/masamune/thor.rb +12 -4
  23. data/lib/masamune/version.rb +1 -1
  24. data/lib/masamune.rb +0 -1
  25. data/spec/masamune/actions/elastic_mapreduce_spec.rb +0 -2
  26. data/spec/masamune/actions/execute_spec.rb +0 -2
  27. data/spec/masamune/actions/hadoop_filesystem_spec.rb +0 -2
  28. data/spec/masamune/actions/hadoop_streaming_spec.rb +0 -2
  29. data/spec/masamune/actions/hive_spec.rb +0 -2
  30. data/spec/masamune/actions/invoke_parallel_spec.rb +0 -2
  31. data/spec/masamune/actions/postgres_admin_spec.rb +0 -2
  32. data/spec/masamune/actions/postgres_spec.rb +0 -2
  33. data/spec/masamune/actions/s3cmd_spec.rb +0 -2
  34. data/spec/masamune/actions/transform_spec.rb +0 -2
  35. data/spec/masamune/after_initialization_callbacks_spec.rb +0 -2
  36. data/spec/masamune/cached_filesystem_spec.rb +0 -2
  37. data/spec/masamune/commands/hadoop_filesystem_spec.rb +0 -2
  38. data/spec/masamune/commands/hadoop_streaming_spec.rb +0 -2
  39. data/spec/masamune/commands/hive_spec.rb +0 -2
  40. data/spec/masamune/commands/postgres_admin_spec.rb +0 -2
  41. data/spec/masamune/commands/postgres_spec.rb +0 -2
  42. data/spec/masamune/commands/retry_with_backoff_spec.rb +0 -2
  43. data/spec/masamune/commands/s3cmd_spec.rb +0 -2
  44. data/spec/masamune/commands/shell_spec.rb +0 -2
  45. data/spec/masamune/configuration_spec.rb +12 -2
  46. data/spec/masamune/data_plan/builder_spec.rb +0 -2
  47. data/spec/masamune/data_plan/elem_spec.rb +73 -5
  48. data/spec/masamune/data_plan/engine_spec.rb +0 -2
  49. data/spec/masamune/data_plan/rule_spec.rb +51 -6
  50. data/spec/masamune/data_plan/set_spec.rb +2 -5
  51. data/spec/masamune/environment_spec.rb +0 -2
  52. data/spec/masamune/filesystem_spec.rb +33 -4
  53. data/spec/masamune/helpers/postgres_spec.rb +0 -2
  54. data/spec/masamune/rspec/job_fixture_spec.rb +365 -0
  55. data/spec/masamune/rspec/shared_example_group_spec.rb +73 -0
  56. data/spec/masamune/schema/catalog_spec.rb +14 -2
  57. data/spec/masamune/schema/column_spec.rb +0 -2
  58. data/spec/masamune/schema/dimension_spec.rb +0 -2
  59. data/spec/masamune/schema/fact_spec.rb +0 -2
  60. data/spec/masamune/schema/map_spec.rb +51 -2
  61. data/spec/masamune/schema/row_spec.rb +0 -2
  62. data/spec/masamune/schema/store_spec.rb +23 -2
  63. data/spec/masamune/schema/table_spec.rb +0 -2
  64. data/spec/masamune/string_format_spec.rb +0 -2
  65. data/spec/masamune/tasks/dump_thor_spec.rb +0 -3
  66. data/spec/masamune/tasks/elastic_mapreduce_thor_spec.rb +0 -3
  67. data/spec/masamune/tasks/hive_thor_spec.rb +0 -3
  68. data/spec/masamune/tasks/postgres_thor_spec.rb +0 -3
  69. data/spec/masamune/tasks/shell_thor_spec.rb +0 -3
  70. data/spec/masamune/template_spec.rb +0 -2
  71. data/spec/masamune/thor_spec.rb +53 -8
  72. data/spec/masamune/transform/bulk_upsert.dimension_spec.rb +0 -2
  73. data/spec/masamune/transform/consolidate_dimension_spec.rb +0 -2
  74. data/spec/masamune/transform/deduplicate_dimension_spec.rb +0 -2
  75. data/spec/masamune/transform/define_schema_spec.rb +0 -2
  76. data/spec/masamune/transform/define_table.dimension_spec.rb +0 -2
  77. data/spec/masamune/transform/define_table.fact_spec.rb +0 -2
  78. data/spec/masamune/transform/define_table.table_spec.rb +0 -2
  79. data/spec/masamune/transform/denormalize_table_spec.rb +0 -2
  80. data/spec/masamune/transform/insert_reference_values.dimension_spec.rb +0 -2
  81. data/spec/masamune/transform/insert_reference_values.fact_spec.rb +0 -2
  82. data/spec/masamune/transform/load_dimension_spec.rb +0 -2
  83. data/spec/masamune/transform/load_fact_spec.rb +0 -2
  84. data/spec/masamune/transform/relabel_dimension_spec.rb +0 -2
  85. data/spec/masamune/transform/rollup_fact_spec.rb +0 -2
  86. data/spec/masamune/transform/snapshot_dimension_spec.rb +0 -2
  87. data/spec/masamune/transform/stage_dimension_spec.rb +0 -2
  88. data/spec/masamune/transform/stage_fact_spec.rb +0 -2
  89. data/spec/masamune_spec.rb +0 -2
  90. data/spec/spec_helper.rb +2 -0
  91. data/spec/support/masamune/job_example_group.rb +62 -0
  92. data/spec/support/masamune/job_fixture.rb +137 -0
  93. data/spec/support/masamune/shared_example_group.rb +203 -0
  94. data/spec/support/masamune/step_example_group.rb +68 -0
  95. data/spec/support/masamune/step_fixture.rb +91 -0
  96. data/{lib/masamune/thor_loader.rb → spec/support/masamune/task_example_group.rb} +33 -10
  97. data/spec/support/rspec/example/action_example_group.rb +1 -1
  98. metadata +32 -3
data/lib/masamune/thor.rb CHANGED
@@ -72,11 +72,14 @@ module Masamune
72
72
  raise e unless %w(SIGHUP SIGTERM).include?(e.to_s)
73
73
  instance.logger.debug("Exiting at user request on #{e.to_s}")
74
74
  exit 0
75
+ rescue ::Thor::MalformattedArgumentError, ::Thor::RequiredArgumentMissingError => e
76
+ raise e
75
77
  rescue => e
76
78
  instance.logger.error("#{e.message} (#{e.class}) backtrace:")
77
79
  e.backtrace.each { |x| instance.logger.error(x) }
80
+ $stderr.puts e.to_s
78
81
  $stderr.puts "For complete debug log see: #{instance.log_file_name.to_s}"
79
- raise e
82
+ exit 1
80
83
  end
81
84
  end
82
85
 
@@ -96,8 +99,7 @@ module Masamune
96
99
  class_option :quiet, :type => :boolean, :aliases => '-q', :desc => 'Suppress all output', :default => false
97
100
  class_option :verbose, :type => :boolean, :aliases => '-v', :desc => 'Print command execution information', :default => false
98
101
  class_option :debug, :type => :boolean, :aliases => '-d', :desc => 'Print debugging information', :default => false
99
- class_option :no_op, :type => :boolean, :desc => 'Do not execute commands that modify state', :default => false
100
- class_option :dry_run, :type => :boolean, :aliases => '-n', :desc => 'Combination of --no-op and --verbose', :default => false
102
+ class_option :dry_run, :type => :boolean, :aliases => '-n', :desc => 'Do not execute commands that modify state', :default => false
101
103
  class_option :config, :desc => 'Configuration file'
102
104
  class_option :version, :desc => 'Print version and exit', :type => :boolean
103
105
  class_option :lock, :desc => 'Optional job lock name', :type => :string
@@ -111,6 +113,8 @@ module Masamune
111
113
  self.current_command_name = current_namespace ? current_namespace + ':' + current_task_name : current_task_name
112
114
  self.class.instance = self
113
115
 
116
+ define_current_dir
117
+
114
118
  if _options.is_a?(Array)
115
119
  _options, self.extra = self.class.parse_extra(_options)
116
120
  end
@@ -137,7 +141,6 @@ module Masamune
137
141
  config.quiet = options[:quiet]
138
142
  config.verbose = options[:verbose] || options[:dry_run]
139
143
  config.debug = options[:debug]
140
- config.no_op = options[:no_op] || options[:dry_run]
141
144
  config.dry_run = options[:dry_run]
142
145
  config.lock = options[:lock]
143
146
 
@@ -195,6 +198,11 @@ module Masamune
195
198
 
196
199
  private
197
200
 
201
+ def define_current_dir
202
+ return unless current_task_name
203
+ filesystem.add_path(:current_dir, File.dirname(method(current_task_name).source_location.first))
204
+ end
205
+
198
206
  def display_help?
199
207
  options[:help] || current_task_name == 'help'
200
208
  end
@@ -21,5 +21,5 @@
21
21
  # THE SOFTWARE.
22
22
 
23
23
  module Masamune
24
- VERSION = '0.14.0'
24
+ VERSION = '0.15.0'
25
25
  end
data/lib/masamune.rb CHANGED
@@ -30,7 +30,6 @@ module Masamune
30
30
  require 'masamune/last_element'
31
31
  require 'masamune/configuration'
32
32
  require 'masamune/thor'
33
- require 'masamune/thor_loader'
34
33
  require 'masamune/filesystem'
35
34
  require 'masamune/cached_filesystem'
36
35
  require 'masamune/method_logger'
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Actions::ElasticMapreduce do
26
24
  let(:klass) do
27
25
  Class.new do
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Actions::Execute do
26
24
  let(:klass) do
27
25
  Class.new do
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Actions::HadoopFilesystem do
26
24
  let(:klass) do
27
25
  Class.new do
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Actions::HadoopStreaming do
26
24
  let(:klass) do
27
25
  Class.new do
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Actions::Hive do
26
24
  let(:klass) do
27
25
  Class.new do
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Actions::InvokeParallel do
26
24
  let(:klass) do
27
25
  Class.new(Thor) do
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Actions::PostgresAdmin do
26
24
  let(:klass) do
27
25
  Class.new do
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Actions::Postgres do
26
24
  let(:filesystem) { Masamune::MockFilesystem.new }
27
25
 
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Actions::S3Cmd do
26
24
  let(:klass) do
27
25
  Class.new do
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Actions::Transform do
26
24
  let(:environment) { double }
27
25
  let(:catalog) { Masamune::Schema::Catalog.new(environment) }
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::AfterInitializeCallbacks do
26
24
  let(:klass) do
27
25
  Class.new do
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::CachedFilesystem do
26
24
  let(:filesystem) { Masamune::MockFilesystem.new }
27
25
  let(:cached_filesystem) { Masamune::CachedFilesystem.new(filesystem) }
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Commands::HadoopFilesystem do
26
24
  let(:configuration) { {options: options} }
27
25
  let(:options) { [] }
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Commands::HadoopStreaming do
26
24
  let(:filesystem) { Masamune::MockFilesystem.new }
27
25
 
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Commands::Hive do
26
24
  let(:filesystem) { Masamune::MockFilesystem.new }
27
25
  let(:configuration) { {:options => options} }
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Commands::PostgresAdmin do
26
24
  let(:configuration) { {:create_db_path => 'createdb', :drop_db_path => 'dropdb', :hostname => 'localhost', :username => 'postgres'} }
27
25
  let(:attrs) { {} }
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Commands::Postgres do
26
24
  let(:configuration) { {:path => 'psql', :database => 'postgres', :options => options} }
27
25
  let(:options) { [] }
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Commands::RetryWithBackoff do
26
24
  let(:options) { {retries: retries, backoff: 0} }
27
25
  let(:delegate) { double }
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Commands::S3Cmd do
26
24
  let(:configuration) { {:options => options} }
27
25
  let(:options) { [] }
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Commands::Shell do
26
24
  let(:input) { nil }
27
25
  let(:options) { {fail_fast: false} }
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Configuration do
26
24
  let(:environment) { Masamune::Environment.new }
27
25
  let(:instance) { described_class.new(environment) }
@@ -36,6 +34,18 @@ describe Masamune::Configuration do
36
34
  it { is_expected.to match(%r{config/masamune\.yml\.erb\Z}) }
37
35
  end
38
36
 
37
+ describe '#as_options' do
38
+ subject { instance.as_options }
39
+ it { is_expected.to eq([]) }
40
+
41
+ context 'with dry_run: true and debug: true' do
42
+ before do
43
+ instance.debug = instance.dry_run = true
44
+ end
45
+ it { is_expected.to eq(['--debug', '--dry-run']) }
46
+ end
47
+ end
48
+
39
49
  describe '#bind_template' do
40
50
  let(:section) { nil }
41
51
  let(:template) { nil }
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::DataPlan::Builder do
26
24
  describe '#build' do
27
25
  subject(:engine) do
@@ -20,14 +20,20 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::DataPlan::Elem do
24
+ let(:filesystem) { Masamune::MockFilesystem.new }
25
+ let(:environment) { Masamune::Environment.new }
26
26
  let(:engine) { Masamune::DataPlan::Engine.new }
27
+
28
+ before do
29
+ environment.filesystem = filesystem
30
+ engine.environment = environment
31
+ end
32
+
27
33
  let(:name) { 'primary' }
28
34
  let(:type) { :target }
29
- let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: 'report/%Y-%m-%d/%H'}) }
30
- let(:other_rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: 'log/%Y%m%d.*.log'}) }
35
+ let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: '/report/%Y-%m-%d/%H'}) }
36
+ let(:other_rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: '/log/%Y%m%d.*.log'}) }
31
37
 
32
38
  let(:start_time) { DateTime.civil(2013,07,19,11,07) }
33
39
  let(:other_start_time) { DateTime.civil(2013,07,20,0,0) }
@@ -41,7 +47,7 @@ describe Masamune::DataPlan::Elem do
41
47
  subject do
42
48
  instance.path
43
49
  end
44
- it { is_expected.to eq('report/2013-07-19/11') }
50
+ it { is_expected.to eq('/report/2013-07-19/11') }
45
51
  end
46
52
 
47
53
  describe '#==' do
@@ -99,4 +105,66 @@ describe Masamune::DataPlan::Elem do
99
105
  it { is_expected.to eq(early) }
100
106
  end
101
107
  end
108
+
109
+ describe '#explode' do
110
+ subject { instance.explode.map(&:path) }
111
+
112
+ context 'with free path and existing files' do
113
+ let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: '/report/%Y-%m-%d/%H'}) }
114
+ before do
115
+ engine.filesystem.touch!('/report/2013-07-19/11/part-0000')
116
+ end
117
+ it { is_expected.to include '/report/2013-07-19/11' }
118
+ end
119
+
120
+ context 'with free path and missing files' do
121
+ let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: '/report/%Y-%m-%d/%H'}) }
122
+ it { is_expected.to be_empty }
123
+ end
124
+
125
+ context 'with bound path and existing files' do
126
+ let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: '/report/file'}) }
127
+ before do
128
+ engine.filesystem.touch!('/report/file')
129
+ end
130
+ it { is_expected.to include '/report/file' }
131
+ end
132
+
133
+ context 'with bound path and missing files' do
134
+ let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: '/report/file'}) }
135
+ it { is_expected.to be_empty }
136
+ end
137
+
138
+ context 'with free table and existing table' do
139
+ let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {table: 'visits_fact', partition: 'y%Y%m'}) }
140
+ before do
141
+ expect(instance.rule.engine.postgres_helper).to receive(:table_exists?).and_return(true)
142
+ end
143
+ it { is_expected.to include 'visits_fact_y201307' }
144
+ end
145
+
146
+ context 'with free table and missing table' do
147
+ let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {table: 'visits_fact', partition: 'y%Y%m'}) }
148
+ before do
149
+ expect(instance.rule.engine.postgres_helper).to receive(:table_exists?).and_return(false)
150
+ end
151
+ it { is_expected.to be_empty }
152
+ end
153
+
154
+ context 'with bound table and existing table' do
155
+ let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {table: 'visits_fact'}) }
156
+ before do
157
+ expect(instance.rule.engine.postgres_helper).to receive(:table_exists?).and_return(true)
158
+ end
159
+ it { is_expected.to include 'visits_fact' }
160
+ end
161
+
162
+ context 'with bound table and missing table' do
163
+ let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {table: 'visits_fact'}) }
164
+ before do
165
+ expect(instance.rule.engine.postgres_helper).to receive(:table_exists?).and_return(false)
166
+ end
167
+ it { is_expected.to be_empty }
168
+ end
169
+ end
102
170
  end
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::DataPlan::Engine do
26
24
  let(:filesystem) { Masamune::MockFilesystem.new }
27
25
  let(:environment) { Masamune::Environment.new }
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::DataPlan::Rule do
26
24
  let(:engine) { Masamune::DataPlan::Engine.new }
27
25
  let(:name) { 'primary' }
@@ -138,6 +136,27 @@ describe Masamune::DataPlan::Rule do
138
136
  let(:input) { instance.bind_input(prev_input) }
139
137
  it { is_expected.to eq(input) }
140
138
  end
139
+
140
+ context 'with wildcard pattern' do
141
+ let(:pattern) { 'requests/y=%Y/m=%-m/d=%-d/h=%-k/*' }
142
+ let(:input) { 'requests/y=2013/m=4/d=30/h=20/part-00000' }
143
+ let(:output_date) { DateTime.civil(2013,04,30,20) }
144
+
145
+ describe '#path' do
146
+ subject { elem.path }
147
+ it { is_expected.to eq(input) }
148
+ end
149
+
150
+ describe '#start_time' do
151
+ subject { elem.start_time }
152
+ it { is_expected.to eq(output_date) }
153
+ end
154
+
155
+ describe '#stop_time' do
156
+ subject { elem.stop_time }
157
+ it { is_expected.to eq(output_date.to_time + 1.hour) }
158
+ end
159
+ end
141
160
  end
142
161
 
143
162
  describe '#unify' do
@@ -200,7 +219,7 @@ describe Masamune::DataPlan::Rule do
200
219
  it { is_expected.to eq(true) }
201
220
  end
202
221
 
203
- context 'with alternative hour' do
222
+ context 'with another alternative hour' do
204
223
  let(:pattern) { 'requests/y=%Y/m=%-m/d=%-d/h=%-k' }
205
224
  let(:input) { 'requests/y=2013/m=4/d=30/h=20' }
206
225
  it { is_expected.to eq(true) }
@@ -212,9 +231,9 @@ describe Masamune::DataPlan::Rule do
212
231
  it { is_expected.to eq(true) }
213
232
  end
214
233
 
215
- context 'with unix timestamp pattern' do
216
- let(:pattern) { 'request_logs/%H-s.log' }
217
- let(:input) { 'request_logs/1374192000.log' }
234
+ context 'with wildcard input' do
235
+ let(:pattern) { 'requests/y=%Y/m=%-m/d=%-d/h=%-k' }
236
+ let(:input) { 'requests/y=2013/m=4/d=30/h=20/*' }
218
237
  it { is_expected.to eq(true) }
219
238
  end
220
239
  end
@@ -419,4 +438,30 @@ describe Masamune::DataPlan::Rule do
419
438
  end
420
439
  end
421
440
  end
441
+
442
+ describe '#free? ' do
443
+ subject { instance.free? }
444
+ context 'with rule that contains free variables' do
445
+ let(:pattern) { 'report/%Y-%m-%d/%H' }
446
+ it { is_expected.to be(true) }
447
+ end
448
+
449
+ context 'with rule that does not contain free variables' do
450
+ let(:pattern) { 'report/file' }
451
+ it { is_expected.to be(false) }
452
+ end
453
+ end
454
+
455
+ describe '#bound? ' do
456
+ subject { instance.bound? }
457
+ context 'with rule that contains free variables' do
458
+ let(:pattern) { 'report/%Y-%m-%d/%H' }
459
+ it { is_expected.to be(false) }
460
+ end
461
+
462
+ context 'with rule that does not contain free variables' do
463
+ let(:pattern) { 'report/file' }
464
+ it { is_expected.to be(true) }
465
+ end
466
+ end
422
467
  end
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::DataPlan::Set do
26
24
  let(:fs) { Masamune::MockFilesystem.new }
27
25
  let!(:engine) { Masamune::DataPlan::Engine.new }
@@ -243,13 +241,12 @@ describe Masamune::DataPlan::Set do
243
241
  it { expect(stale_targets).to include '/table/y=2013/m=01/d=01' }
244
242
  end
245
243
 
246
- context 'when some stale targets (tie breaker)' do
244
+ context 'when none stale targets (tie breaker)' do
247
245
  before do
248
246
  fs.touch!('/log/20130101.random_1.log', mtime: present_time)
249
247
  end
250
248
 
251
- it { expect(stale_targets.count).to eq(1) }
252
- it { expect(stale_targets).to include '/table/y=2013/m=01/d=01' }
249
+ it { expect(stale_targets).to be_empty }
253
250
  end
254
251
 
255
252
  context 'when all stale targets' do
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Environment do
26
24
  let(:instance) { described_class.new }
27
25
  let(:run_dir) { Dir.mktmpdir('masamune') }
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  require 'securerandom'
26
24
 
27
25
  # NOTE when operating between hdfs and s3, hadoop fs requires s3n URI
@@ -62,6 +60,10 @@ shared_examples_for 'Filesystem' do
62
60
  it { expect(instance.get_path(:home_dir, '/a/b', 'c')).to eq('/home/a/b/c') }
63
61
  end
64
62
 
63
+ context 'with extra options' do
64
+ it { expect(instance.get_path(:home_dir, '/a/b', 'c', mkdir: true)).to eq('/home/a/b/c') }
65
+ end
66
+
65
67
  context 'with parameter substitution' do
66
68
  before do
67
69
  instance.configuration.params[:user] = 'zombo'
@@ -373,6 +375,14 @@ shared_examples_for 'Filesystem' do
373
375
  it { is_expected.to eq(true) }
374
376
  end
375
377
 
378
+ context 'local missing file after glob' do
379
+ before do
380
+ instance.glob(new_file + '/*')
381
+ end
382
+ subject { instance.exists?(new_file) }
383
+ it { is_expected.to eq(false) }
384
+ end
385
+
376
386
  context 'hdfs existing file' do
377
387
  before do
378
388
  expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.dirname(old_dir) + '/*', safe: true).at_most(:once).
@@ -567,6 +577,15 @@ shared_examples_for 'Filesystem' do
567
577
  it { is_expected.to eq(true) }
568
578
  end
569
579
 
580
+ context 'local existing directory' do
581
+ subject { Dir.exists?(old_dir) }
582
+ before do
583
+ expect(FileUtils).to receive(:mkdir_p).never
584
+ instance.mkdir!(old_dir)
585
+ end
586
+ it { is_expected.to eq(true) }
587
+ end
588
+
570
589
  context 'hdfs directory' do
571
590
  it do
572
591
  expect(filesystem).to receive(:hadoop_fs).with('-mkdir', '-p', 'file://' + new_dir, 'file://' + other_new_dir).once
@@ -610,8 +629,18 @@ shared_examples_for 'Filesystem' do
610
629
  it { expect { |b| instance.glob(pattern, &b) }.to yield_successive_args(old_dir, old_file) }
611
630
  end
612
631
 
613
- context 'local one matches (with suffix)' do
614
- let(:pattern) { File.join(File.dirname(old_file), '*.txt') }
632
+ context 'local one matches with glob' do
633
+ let(:pattern) { File.join(File.dirname(old_dir), '*') }
634
+ it 'has 1 item' do
635
+ expect(subject.count).to eq(2)
636
+ end
637
+ it { is_expected.to include old_dir }
638
+ it { is_expected.to include old_file }
639
+ it { expect { |b| instance.glob(pattern, &b) }.to yield_successive_args(old_dir, old_file) }
640
+ end
641
+
642
+ context 'local one matches with glob and suffix' do
643
+ let(:pattern) { File.join(File.dirname(old_dir), '*.txt') }
615
644
  it 'has 1 item' do
616
645
  expect(subject.count).to eq(1)
617
646
  end
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Helpers::Postgres do
26
24
  let(:environment) { double }
27
25
  let(:instance) { described_class.new(environment) }