masamune 0.14.0 → 0.15.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (98) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +10 -1
  3. data/Rakefile +37 -0
  4. data/bin/masamune-dump +22 -0
  5. data/bin/masamune-elastic-mapreduce +22 -0
  6. data/bin/masamune-hive +22 -0
  7. data/bin/masamune-psql +22 -0
  8. data/bin/masamune-shell +22 -0
  9. data/lib/masamune/cached_filesystem.rb +1 -1
  10. data/lib/masamune/commands/shell.rb +1 -1
  11. data/lib/masamune/configuration.rb +5 -7
  12. data/lib/masamune/data_plan/elem.rb +15 -7
  13. data/lib/masamune/data_plan/engine.rb +2 -0
  14. data/lib/masamune/data_plan/rule.rb +16 -5
  15. data/lib/masamune/data_plan/set.rb +8 -8
  16. data/lib/masamune/filesystem.rb +12 -6
  17. data/lib/masamune/schema/catalog.rb +6 -6
  18. data/lib/masamune/schema/column.rb +1 -1
  19. data/lib/masamune/schema/map.rb +6 -2
  20. data/lib/masamune/schema/store.rb +31 -3
  21. data/lib/masamune/tasks/shell_thor.rb +1 -1
  22. data/lib/masamune/thor.rb +12 -4
  23. data/lib/masamune/version.rb +1 -1
  24. data/lib/masamune.rb +0 -1
  25. data/spec/masamune/actions/elastic_mapreduce_spec.rb +0 -2
  26. data/spec/masamune/actions/execute_spec.rb +0 -2
  27. data/spec/masamune/actions/hadoop_filesystem_spec.rb +0 -2
  28. data/spec/masamune/actions/hadoop_streaming_spec.rb +0 -2
  29. data/spec/masamune/actions/hive_spec.rb +0 -2
  30. data/spec/masamune/actions/invoke_parallel_spec.rb +0 -2
  31. data/spec/masamune/actions/postgres_admin_spec.rb +0 -2
  32. data/spec/masamune/actions/postgres_spec.rb +0 -2
  33. data/spec/masamune/actions/s3cmd_spec.rb +0 -2
  34. data/spec/masamune/actions/transform_spec.rb +0 -2
  35. data/spec/masamune/after_initialization_callbacks_spec.rb +0 -2
  36. data/spec/masamune/cached_filesystem_spec.rb +0 -2
  37. data/spec/masamune/commands/hadoop_filesystem_spec.rb +0 -2
  38. data/spec/masamune/commands/hadoop_streaming_spec.rb +0 -2
  39. data/spec/masamune/commands/hive_spec.rb +0 -2
  40. data/spec/masamune/commands/postgres_admin_spec.rb +0 -2
  41. data/spec/masamune/commands/postgres_spec.rb +0 -2
  42. data/spec/masamune/commands/retry_with_backoff_spec.rb +0 -2
  43. data/spec/masamune/commands/s3cmd_spec.rb +0 -2
  44. data/spec/masamune/commands/shell_spec.rb +0 -2
  45. data/spec/masamune/configuration_spec.rb +12 -2
  46. data/spec/masamune/data_plan/builder_spec.rb +0 -2
  47. data/spec/masamune/data_plan/elem_spec.rb +73 -5
  48. data/spec/masamune/data_plan/engine_spec.rb +0 -2
  49. data/spec/masamune/data_plan/rule_spec.rb +51 -6
  50. data/spec/masamune/data_plan/set_spec.rb +2 -5
  51. data/spec/masamune/environment_spec.rb +0 -2
  52. data/spec/masamune/filesystem_spec.rb +33 -4
  53. data/spec/masamune/helpers/postgres_spec.rb +0 -2
  54. data/spec/masamune/rspec/job_fixture_spec.rb +365 -0
  55. data/spec/masamune/rspec/shared_example_group_spec.rb +73 -0
  56. data/spec/masamune/schema/catalog_spec.rb +14 -2
  57. data/spec/masamune/schema/column_spec.rb +0 -2
  58. data/spec/masamune/schema/dimension_spec.rb +0 -2
  59. data/spec/masamune/schema/fact_spec.rb +0 -2
  60. data/spec/masamune/schema/map_spec.rb +51 -2
  61. data/spec/masamune/schema/row_spec.rb +0 -2
  62. data/spec/masamune/schema/store_spec.rb +23 -2
  63. data/spec/masamune/schema/table_spec.rb +0 -2
  64. data/spec/masamune/string_format_spec.rb +0 -2
  65. data/spec/masamune/tasks/dump_thor_spec.rb +0 -3
  66. data/spec/masamune/tasks/elastic_mapreduce_thor_spec.rb +0 -3
  67. data/spec/masamune/tasks/hive_thor_spec.rb +0 -3
  68. data/spec/masamune/tasks/postgres_thor_spec.rb +0 -3
  69. data/spec/masamune/tasks/shell_thor_spec.rb +0 -3
  70. data/spec/masamune/template_spec.rb +0 -2
  71. data/spec/masamune/thor_spec.rb +53 -8
  72. data/spec/masamune/transform/bulk_upsert.dimension_spec.rb +0 -2
  73. data/spec/masamune/transform/consolidate_dimension_spec.rb +0 -2
  74. data/spec/masamune/transform/deduplicate_dimension_spec.rb +0 -2
  75. data/spec/masamune/transform/define_schema_spec.rb +0 -2
  76. data/spec/masamune/transform/define_table.dimension_spec.rb +0 -2
  77. data/spec/masamune/transform/define_table.fact_spec.rb +0 -2
  78. data/spec/masamune/transform/define_table.table_spec.rb +0 -2
  79. data/spec/masamune/transform/denormalize_table_spec.rb +0 -2
  80. data/spec/masamune/transform/insert_reference_values.dimension_spec.rb +0 -2
  81. data/spec/masamune/transform/insert_reference_values.fact_spec.rb +0 -2
  82. data/spec/masamune/transform/load_dimension_spec.rb +0 -2
  83. data/spec/masamune/transform/load_fact_spec.rb +0 -2
  84. data/spec/masamune/transform/relabel_dimension_spec.rb +0 -2
  85. data/spec/masamune/transform/rollup_fact_spec.rb +0 -2
  86. data/spec/masamune/transform/snapshot_dimension_spec.rb +0 -2
  87. data/spec/masamune/transform/stage_dimension_spec.rb +0 -2
  88. data/spec/masamune/transform/stage_fact_spec.rb +0 -2
  89. data/spec/masamune_spec.rb +0 -2
  90. data/spec/spec_helper.rb +2 -0
  91. data/spec/support/masamune/job_example_group.rb +62 -0
  92. data/spec/support/masamune/job_fixture.rb +137 -0
  93. data/spec/support/masamune/shared_example_group.rb +203 -0
  94. data/spec/support/masamune/step_example_group.rb +68 -0
  95. data/spec/support/masamune/step_fixture.rb +91 -0
  96. data/{lib/masamune/thor_loader.rb → spec/support/masamune/task_example_group.rb} +33 -10
  97. data/spec/support/rspec/example/action_example_group.rb +1 -1
  98. metadata +32 -3
data/lib/masamune/thor.rb CHANGED
@@ -72,11 +72,14 @@ module Masamune
72
72
  raise e unless %w(SIGHUP SIGTERM).include?(e.to_s)
73
73
  instance.logger.debug("Exiting at user request on #{e.to_s}")
74
74
  exit 0
75
+ rescue ::Thor::MalformattedArgumentError, ::Thor::RequiredArgumentMissingError => e
76
+ raise e
75
77
  rescue => e
76
78
  instance.logger.error("#{e.message} (#{e.class}) backtrace:")
77
79
  e.backtrace.each { |x| instance.logger.error(x) }
80
+ $stderr.puts e.to_s
78
81
  $stderr.puts "For complete debug log see: #{instance.log_file_name.to_s}"
79
- raise e
82
+ exit 1
80
83
  end
81
84
  end
82
85
 
@@ -96,8 +99,7 @@ module Masamune
96
99
  class_option :quiet, :type => :boolean, :aliases => '-q', :desc => 'Suppress all output', :default => false
97
100
  class_option :verbose, :type => :boolean, :aliases => '-v', :desc => 'Print command execution information', :default => false
98
101
  class_option :debug, :type => :boolean, :aliases => '-d', :desc => 'Print debugging information', :default => false
99
- class_option :no_op, :type => :boolean, :desc => 'Do not execute commands that modify state', :default => false
100
- class_option :dry_run, :type => :boolean, :aliases => '-n', :desc => 'Combination of --no-op and --verbose', :default => false
102
+ class_option :dry_run, :type => :boolean, :aliases => '-n', :desc => 'Do not execute commands that modify state', :default => false
101
103
  class_option :config, :desc => 'Configuration file'
102
104
  class_option :version, :desc => 'Print version and exit', :type => :boolean
103
105
  class_option :lock, :desc => 'Optional job lock name', :type => :string
@@ -111,6 +113,8 @@ module Masamune
111
113
  self.current_command_name = current_namespace ? current_namespace + ':' + current_task_name : current_task_name
112
114
  self.class.instance = self
113
115
 
116
+ define_current_dir
117
+
114
118
  if _options.is_a?(Array)
115
119
  _options, self.extra = self.class.parse_extra(_options)
116
120
  end
@@ -137,7 +141,6 @@ module Masamune
137
141
  config.quiet = options[:quiet]
138
142
  config.verbose = options[:verbose] || options[:dry_run]
139
143
  config.debug = options[:debug]
140
- config.no_op = options[:no_op] || options[:dry_run]
141
144
  config.dry_run = options[:dry_run]
142
145
  config.lock = options[:lock]
143
146
 
@@ -195,6 +198,11 @@ module Masamune
195
198
 
196
199
  private
197
200
 
201
+ def define_current_dir
202
+ return unless current_task_name
203
+ filesystem.add_path(:current_dir, File.dirname(method(current_task_name).source_location.first))
204
+ end
205
+
198
206
  def display_help?
199
207
  options[:help] || current_task_name == 'help'
200
208
  end
@@ -21,5 +21,5 @@
21
21
  # THE SOFTWARE.
22
22
 
23
23
  module Masamune
24
- VERSION = '0.14.0'
24
+ VERSION = '0.15.0'
25
25
  end
data/lib/masamune.rb CHANGED
@@ -30,7 +30,6 @@ module Masamune
30
30
  require 'masamune/last_element'
31
31
  require 'masamune/configuration'
32
32
  require 'masamune/thor'
33
- require 'masamune/thor_loader'
34
33
  require 'masamune/filesystem'
35
34
  require 'masamune/cached_filesystem'
36
35
  require 'masamune/method_logger'
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Actions::ElasticMapreduce do
26
24
  let(:klass) do
27
25
  Class.new do
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Actions::Execute do
26
24
  let(:klass) do
27
25
  Class.new do
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Actions::HadoopFilesystem do
26
24
  let(:klass) do
27
25
  Class.new do
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Actions::HadoopStreaming do
26
24
  let(:klass) do
27
25
  Class.new do
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Actions::Hive do
26
24
  let(:klass) do
27
25
  Class.new do
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Actions::InvokeParallel do
26
24
  let(:klass) do
27
25
  Class.new(Thor) do
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Actions::PostgresAdmin do
26
24
  let(:klass) do
27
25
  Class.new do
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Actions::Postgres do
26
24
  let(:filesystem) { Masamune::MockFilesystem.new }
27
25
 
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Actions::S3Cmd do
26
24
  let(:klass) do
27
25
  Class.new do
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Actions::Transform do
26
24
  let(:environment) { double }
27
25
  let(:catalog) { Masamune::Schema::Catalog.new(environment) }
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::AfterInitializeCallbacks do
26
24
  let(:klass) do
27
25
  Class.new do
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::CachedFilesystem do
26
24
  let(:filesystem) { Masamune::MockFilesystem.new }
27
25
  let(:cached_filesystem) { Masamune::CachedFilesystem.new(filesystem) }
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Commands::HadoopFilesystem do
26
24
  let(:configuration) { {options: options} }
27
25
  let(:options) { [] }
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Commands::HadoopStreaming do
26
24
  let(:filesystem) { Masamune::MockFilesystem.new }
27
25
 
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Commands::Hive do
26
24
  let(:filesystem) { Masamune::MockFilesystem.new }
27
25
  let(:configuration) { {:options => options} }
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Commands::PostgresAdmin do
26
24
  let(:configuration) { {:create_db_path => 'createdb', :drop_db_path => 'dropdb', :hostname => 'localhost', :username => 'postgres'} }
27
25
  let(:attrs) { {} }
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Commands::Postgres do
26
24
  let(:configuration) { {:path => 'psql', :database => 'postgres', :options => options} }
27
25
  let(:options) { [] }
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Commands::RetryWithBackoff do
26
24
  let(:options) { {retries: retries, backoff: 0} }
27
25
  let(:delegate) { double }
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Commands::S3Cmd do
26
24
  let(:configuration) { {:options => options} }
27
25
  let(:options) { [] }
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Commands::Shell do
26
24
  let(:input) { nil }
27
25
  let(:options) { {fail_fast: false} }
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Configuration do
26
24
  let(:environment) { Masamune::Environment.new }
27
25
  let(:instance) { described_class.new(environment) }
@@ -36,6 +34,18 @@ describe Masamune::Configuration do
36
34
  it { is_expected.to match(%r{config/masamune\.yml\.erb\Z}) }
37
35
  end
38
36
 
37
+ describe '#as_options' do
38
+ subject { instance.as_options }
39
+ it { is_expected.to eq([]) }
40
+
41
+ context 'with dry_run: true and debug: true' do
42
+ before do
43
+ instance.debug = instance.dry_run = true
44
+ end
45
+ it { is_expected.to eq(['--debug', '--dry-run']) }
46
+ end
47
+ end
48
+
39
49
  describe '#bind_template' do
40
50
  let(:section) { nil }
41
51
  let(:template) { nil }
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::DataPlan::Builder do
26
24
  describe '#build' do
27
25
  subject(:engine) do
@@ -20,14 +20,20 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::DataPlan::Elem do
24
+ let(:filesystem) { Masamune::MockFilesystem.new }
25
+ let(:environment) { Masamune::Environment.new }
26
26
  let(:engine) { Masamune::DataPlan::Engine.new }
27
+
28
+ before do
29
+ environment.filesystem = filesystem
30
+ engine.environment = environment
31
+ end
32
+
27
33
  let(:name) { 'primary' }
28
34
  let(:type) { :target }
29
- let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: 'report/%Y-%m-%d/%H'}) }
30
- let(:other_rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: 'log/%Y%m%d.*.log'}) }
35
+ let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: '/report/%Y-%m-%d/%H'}) }
36
+ let(:other_rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: '/log/%Y%m%d.*.log'}) }
31
37
 
32
38
  let(:start_time) { DateTime.civil(2013,07,19,11,07) }
33
39
  let(:other_start_time) { DateTime.civil(2013,07,20,0,0) }
@@ -41,7 +47,7 @@ describe Masamune::DataPlan::Elem do
41
47
  subject do
42
48
  instance.path
43
49
  end
44
- it { is_expected.to eq('report/2013-07-19/11') }
50
+ it { is_expected.to eq('/report/2013-07-19/11') }
45
51
  end
46
52
 
47
53
  describe '#==' do
@@ -99,4 +105,66 @@ describe Masamune::DataPlan::Elem do
99
105
  it { is_expected.to eq(early) }
100
106
  end
101
107
  end
108
+
109
+ describe '#explode' do
110
+ subject { instance.explode.map(&:path) }
111
+
112
+ context 'with free path and existing files' do
113
+ let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: '/report/%Y-%m-%d/%H'}) }
114
+ before do
115
+ engine.filesystem.touch!('/report/2013-07-19/11/part-0000')
116
+ end
117
+ it { is_expected.to include '/report/2013-07-19/11' }
118
+ end
119
+
120
+ context 'with free path and missing files' do
121
+ let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: '/report/%Y-%m-%d/%H'}) }
122
+ it { is_expected.to be_empty }
123
+ end
124
+
125
+ context 'with bound path and existing files' do
126
+ let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: '/report/file'}) }
127
+ before do
128
+ engine.filesystem.touch!('/report/file')
129
+ end
130
+ it { is_expected.to include '/report/file' }
131
+ end
132
+
133
+ context 'with bound path and missing files' do
134
+ let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {path: '/report/file'}) }
135
+ it { is_expected.to be_empty }
136
+ end
137
+
138
+ context 'with free table and existing table' do
139
+ let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {table: 'visits_fact', partition: 'y%Y%m'}) }
140
+ before do
141
+ expect(instance.rule.engine.postgres_helper).to receive(:table_exists?).and_return(true)
142
+ end
143
+ it { is_expected.to include 'visits_fact_y201307' }
144
+ end
145
+
146
+ context 'with free table and missing table' do
147
+ let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {table: 'visits_fact', partition: 'y%Y%m'}) }
148
+ before do
149
+ expect(instance.rule.engine.postgres_helper).to receive(:table_exists?).and_return(false)
150
+ end
151
+ it { is_expected.to be_empty }
152
+ end
153
+
154
+ context 'with bound table and existing table' do
155
+ let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {table: 'visits_fact'}) }
156
+ before do
157
+ expect(instance.rule.engine.postgres_helper).to receive(:table_exists?).and_return(true)
158
+ end
159
+ it { is_expected.to include 'visits_fact' }
160
+ end
161
+
162
+ context 'with bound table and missing table' do
163
+ let(:rule) { Masamune::DataPlan::Rule.new(engine, name, type, {table: 'visits_fact'}) }
164
+ before do
165
+ expect(instance.rule.engine.postgres_helper).to receive(:table_exists?).and_return(false)
166
+ end
167
+ it { is_expected.to be_empty }
168
+ end
169
+ end
102
170
  end
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::DataPlan::Engine do
26
24
  let(:filesystem) { Masamune::MockFilesystem.new }
27
25
  let(:environment) { Masamune::Environment.new }
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::DataPlan::Rule do
26
24
  let(:engine) { Masamune::DataPlan::Engine.new }
27
25
  let(:name) { 'primary' }
@@ -138,6 +136,27 @@ describe Masamune::DataPlan::Rule do
138
136
  let(:input) { instance.bind_input(prev_input) }
139
137
  it { is_expected.to eq(input) }
140
138
  end
139
+
140
+ context 'with wildcard pattern' do
141
+ let(:pattern) { 'requests/y=%Y/m=%-m/d=%-d/h=%-k/*' }
142
+ let(:input) { 'requests/y=2013/m=4/d=30/h=20/part-00000' }
143
+ let(:output_date) { DateTime.civil(2013,04,30,20) }
144
+
145
+ describe '#path' do
146
+ subject { elem.path }
147
+ it { is_expected.to eq(input) }
148
+ end
149
+
150
+ describe '#start_time' do
151
+ subject { elem.start_time }
152
+ it { is_expected.to eq(output_date) }
153
+ end
154
+
155
+ describe '#stop_time' do
156
+ subject { elem.stop_time }
157
+ it { is_expected.to eq(output_date.to_time + 1.hour) }
158
+ end
159
+ end
141
160
  end
142
161
 
143
162
  describe '#unify' do
@@ -200,7 +219,7 @@ describe Masamune::DataPlan::Rule do
200
219
  it { is_expected.to eq(true) }
201
220
  end
202
221
 
203
- context 'with alternative hour' do
222
+ context 'with another alternative hour' do
204
223
  let(:pattern) { 'requests/y=%Y/m=%-m/d=%-d/h=%-k' }
205
224
  let(:input) { 'requests/y=2013/m=4/d=30/h=20' }
206
225
  it { is_expected.to eq(true) }
@@ -212,9 +231,9 @@ describe Masamune::DataPlan::Rule do
212
231
  it { is_expected.to eq(true) }
213
232
  end
214
233
 
215
- context 'with unix timestamp pattern' do
216
- let(:pattern) { 'request_logs/%H-s.log' }
217
- let(:input) { 'request_logs/1374192000.log' }
234
+ context 'with wildcard input' do
235
+ let(:pattern) { 'requests/y=%Y/m=%-m/d=%-d/h=%-k' }
236
+ let(:input) { 'requests/y=2013/m=4/d=30/h=20/*' }
218
237
  it { is_expected.to eq(true) }
219
238
  end
220
239
  end
@@ -419,4 +438,30 @@ describe Masamune::DataPlan::Rule do
419
438
  end
420
439
  end
421
440
  end
441
+
442
+ describe '#free? ' do
443
+ subject { instance.free? }
444
+ context 'with rule that contains free variables' do
445
+ let(:pattern) { 'report/%Y-%m-%d/%H' }
446
+ it { is_expected.to be(true) }
447
+ end
448
+
449
+ context 'with rule that does not contain free variables' do
450
+ let(:pattern) { 'report/file' }
451
+ it { is_expected.to be(false) }
452
+ end
453
+ end
454
+
455
+ describe '#bound? ' do
456
+ subject { instance.bound? }
457
+ context 'with rule that contains free variables' do
458
+ let(:pattern) { 'report/%Y-%m-%d/%H' }
459
+ it { is_expected.to be(false) }
460
+ end
461
+
462
+ context 'with rule that does not contain free variables' do
463
+ let(:pattern) { 'report/file' }
464
+ it { is_expected.to be(true) }
465
+ end
466
+ end
422
467
  end
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::DataPlan::Set do
26
24
  let(:fs) { Masamune::MockFilesystem.new }
27
25
  let!(:engine) { Masamune::DataPlan::Engine.new }
@@ -243,13 +241,12 @@ describe Masamune::DataPlan::Set do
243
241
  it { expect(stale_targets).to include '/table/y=2013/m=01/d=01' }
244
242
  end
245
243
 
246
- context 'when some stale targets (tie breaker)' do
244
+ context 'when none stale targets (tie breaker)' do
247
245
  before do
248
246
  fs.touch!('/log/20130101.random_1.log', mtime: present_time)
249
247
  end
250
248
 
251
- it { expect(stale_targets.count).to eq(1) }
252
- it { expect(stale_targets).to include '/table/y=2013/m=01/d=01' }
249
+ it { expect(stale_targets).to be_empty }
253
250
  end
254
251
 
255
252
  context 'when all stale targets' do
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Environment do
26
24
  let(:instance) { described_class.new }
27
25
  let(:run_dir) { Dir.mktmpdir('masamune') }
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  require 'securerandom'
26
24
 
27
25
  # NOTE when operating between hdfs and s3, hadoop fs requires s3n URI
@@ -62,6 +60,10 @@ shared_examples_for 'Filesystem' do
62
60
  it { expect(instance.get_path(:home_dir, '/a/b', 'c')).to eq('/home/a/b/c') }
63
61
  end
64
62
 
63
+ context 'with extra options' do
64
+ it { expect(instance.get_path(:home_dir, '/a/b', 'c', mkdir: true)).to eq('/home/a/b/c') }
65
+ end
66
+
65
67
  context 'with parameter substitution' do
66
68
  before do
67
69
  instance.configuration.params[:user] = 'zombo'
@@ -373,6 +375,14 @@ shared_examples_for 'Filesystem' do
373
375
  it { is_expected.to eq(true) }
374
376
  end
375
377
 
378
+ context 'local missing file after glob' do
379
+ before do
380
+ instance.glob(new_file + '/*')
381
+ end
382
+ subject { instance.exists?(new_file) }
383
+ it { is_expected.to eq(false) }
384
+ end
385
+
376
386
  context 'hdfs existing file' do
377
387
  before do
378
388
  expect(filesystem).to receive(:hadoop_fs).with('-ls', '-R', 'file://' + File.dirname(old_dir) + '/*', safe: true).at_most(:once).
@@ -567,6 +577,15 @@ shared_examples_for 'Filesystem' do
567
577
  it { is_expected.to eq(true) }
568
578
  end
569
579
 
580
+ context 'local existing directory' do
581
+ subject { Dir.exists?(old_dir) }
582
+ before do
583
+ expect(FileUtils).to receive(:mkdir_p).never
584
+ instance.mkdir!(old_dir)
585
+ end
586
+ it { is_expected.to eq(true) }
587
+ end
588
+
570
589
  context 'hdfs directory' do
571
590
  it do
572
591
  expect(filesystem).to receive(:hadoop_fs).with('-mkdir', '-p', 'file://' + new_dir, 'file://' + other_new_dir).once
@@ -610,8 +629,18 @@ shared_examples_for 'Filesystem' do
610
629
  it { expect { |b| instance.glob(pattern, &b) }.to yield_successive_args(old_dir, old_file) }
611
630
  end
612
631
 
613
- context 'local one matches (with suffix)' do
614
- let(:pattern) { File.join(File.dirname(old_file), '*.txt') }
632
+ context 'local one matches with glob' do
633
+ let(:pattern) { File.join(File.dirname(old_dir), '*') }
634
+ it 'has 1 item' do
635
+ expect(subject.count).to eq(2)
636
+ end
637
+ it { is_expected.to include old_dir }
638
+ it { is_expected.to include old_file }
639
+ it { expect { |b| instance.glob(pattern, &b) }.to yield_successive_args(old_dir, old_file) }
640
+ end
641
+
642
+ context 'local one matches with glob and suffix' do
643
+ let(:pattern) { File.join(File.dirname(old_dir), '*.txt') }
615
644
  it 'has 1 item' do
616
645
  expect(subject.count).to eq(1)
617
646
  end
@@ -20,8 +20,6 @@
20
20
  # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
21
21
  # THE SOFTWARE.
22
22
 
23
- require 'spec_helper'
24
-
25
23
  describe Masamune::Helpers::Postgres do
26
24
  let(:environment) { double }
27
25
  let(:instance) { described_class.new(environment) }