masamune 0.17.5 → 0.17.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/masamune/actions/transform.rb +10 -10
- data/lib/masamune/commands/postgres.rb +10 -0
- data/lib/masamune/commands/shell.rb +9 -1
- data/lib/masamune/schema/map.rb +56 -16
- data/lib/masamune/thor.rb +1 -2
- data/lib/masamune/version.rb +1 -1
- data/spec/masamune/actions/transform_spec.rb +11 -6
- data/spec/masamune/commands/postgres_spec.rb +23 -0
- data/spec/masamune/commands/shell_spec.rb +13 -0
- data/spec/masamune/schema/map_spec.rb +201 -3
- data/spec/masamune/thor_spec.rb +2 -1
- data/spec/support/masamune/shared_example_group.rb +0 -2
- data/spec/support/masamune/task_example_group.rb +0 -1
- metadata +31 -31
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c29146a9633b2cf7760df9db790297c0889d856c
|
4
|
+
data.tar.gz: d655ab1bd2088a9535b132dba044fe94a9c6eaa8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: bd15f96322e460a7e01c88465d4b245ee27242729c06b0567ad788fd5d02c57a72aea02b5e7cf58d9fe5217018e33b271f6723c8e522fcea37fbbb95b56afbbd
|
7
|
+
data.tar.gz: 79c5a0895b9a64ff4a7b0ce388fc344a05e21b10c674b1069fe671ce486e55e2eaddb8e81792f513da95dec966f4e9380b537627faa93e75f19828ebf90df414
|
@@ -46,33 +46,33 @@ module Masamune::Actions
|
|
46
46
|
|
47
47
|
FILE_MODE = 0777 - File.umask
|
48
48
|
|
49
|
-
def load_dimension(source_files, source, target)
|
49
|
+
def load_dimension(source_files, source, target, options = {})
|
50
50
|
optional_apply_map(source_files, source, target) do |intermediate_files, intermediate|
|
51
51
|
transform = Wrapper.load_dimension(intermediate_files, intermediate, target)
|
52
|
-
postgres file: transform.to_file, debug: (source.debug || target.debug || intermediate.debug)
|
52
|
+
postgres file: transform.to_file, debug: (source.debug || target.debug || intermediate.debug), **options
|
53
53
|
end
|
54
54
|
end
|
55
55
|
|
56
|
-
def consolidate_dimension(target)
|
56
|
+
def consolidate_dimension(target, options = {})
|
57
57
|
transform = Wrapper.consolidate_dimension(target)
|
58
|
-
postgres file: transform.to_file, debug: target.debug
|
58
|
+
postgres file: transform.to_file, debug: target.debug, **options
|
59
59
|
end
|
60
60
|
|
61
|
-
def relabel_dimension(target)
|
61
|
+
def relabel_dimension(target, options = {})
|
62
62
|
transform = Wrapper.relabel_dimension(target)
|
63
|
-
postgres file: transform.to_file, debug: target.debug
|
63
|
+
postgres file: transform.to_file, debug: target.debug, **options
|
64
64
|
end
|
65
65
|
|
66
|
-
def load_fact(source_files, source, target, date)
|
66
|
+
def load_fact(source_files, source, target, date, options = {})
|
67
67
|
optional_apply_map(source_files, source, target) do |intermediate_files, intermediate|
|
68
68
|
transform = Wrapper.load_fact(intermediate_files, intermediate, target, date)
|
69
|
-
postgres file: transform.to_file, debug: (source.debug || target.debug || intermediate.debug)
|
69
|
+
postgres file: transform.to_file, debug: (source.debug || target.debug || intermediate.debug), **options
|
70
70
|
end
|
71
71
|
end
|
72
72
|
|
73
|
-
def rollup_fact(source, target, date)
|
73
|
+
def rollup_fact(source, target, date, options = {})
|
74
74
|
transform = Wrapper.rollup_fact(source, target, date)
|
75
|
-
postgres file: transform.to_file, debug: (source.debug || target.debug)
|
75
|
+
postgres file: transform.to_file, debug: (source.debug || target.debug), **options
|
76
76
|
end
|
77
77
|
|
78
78
|
private
|
@@ -56,6 +56,7 @@ module Masamune::Commands
|
|
56
56
|
instance_variable_set("@#{name}", value)
|
57
57
|
end
|
58
58
|
raise ArgumentError, 'Cannot specify both file and exec' if @file && @exec
|
59
|
+
@error = nil
|
59
60
|
end
|
60
61
|
|
61
62
|
def stdin
|
@@ -109,6 +110,15 @@ module Masamune::Commands
|
|
109
110
|
end
|
110
111
|
end
|
111
112
|
|
113
|
+
def handle_stderr(line, line_no)
|
114
|
+
@error = line.split(/ERROR:\s*/).last if line =~ /ERROR:/
|
115
|
+
logger.debug(line)
|
116
|
+
end
|
117
|
+
|
118
|
+
def failure_message(status)
|
119
|
+
@error || 'psql failed without error'
|
120
|
+
end
|
121
|
+
|
112
122
|
def prompt
|
113
123
|
@database + '=>'
|
114
124
|
end
|
@@ -189,7 +189,15 @@ module Masamune::Commands
|
|
189
189
|
if @delegate.respond_to?(:handle_failure)
|
190
190
|
@delegate.handle_failure(status)
|
191
191
|
end
|
192
|
-
raise
|
192
|
+
raise failure_message(status) if fail_fast
|
193
|
+
end
|
194
|
+
|
195
|
+
def failure_message(status)
|
196
|
+
if @delegate.respond_to?(:failure_message)
|
197
|
+
@delegate.failure_message(status)
|
198
|
+
else
|
199
|
+
"fail_fast: #{command_args.join(' ')}"
|
200
|
+
end
|
193
201
|
end
|
194
202
|
|
195
203
|
private
|
data/lib/masamune/schema/map.rb
CHANGED
@@ -89,11 +89,11 @@ module Masamune::Schema
|
|
89
89
|
|
90
90
|
def_delegators :@io, :flush, :path
|
91
91
|
|
92
|
-
def initialize(
|
92
|
+
def initialize(map, table)
|
93
|
+
@map = map
|
93
94
|
@table = table
|
94
95
|
@store = table.store
|
95
|
-
@
|
96
|
-
@options = options
|
96
|
+
@line = 0
|
97
97
|
end
|
98
98
|
|
99
99
|
def bind(io)
|
@@ -106,21 +106,46 @@ module Masamune::Schema
|
|
106
106
|
CSV.parse(JSONEncoder.new(@io, @store), options.merge(headers: @store.headers || @table.columns.keys)) do |data|
|
107
107
|
next if data.to_s =~ /\A#/
|
108
108
|
yield safe_row(data)
|
109
|
+
@line += 1
|
109
110
|
end
|
110
111
|
end
|
111
112
|
|
112
113
|
def append(data)
|
113
114
|
raise 'must call Buffer#bind first' unless @io
|
114
115
|
row = Masamune::Schema::Row.new(parent: @table, values: data.to_hash)
|
115
|
-
write_headers = @store.headers && @
|
116
|
+
write_headers = @store.headers && @line < 1
|
116
117
|
@csv ||= CSV.new(@io, options.merge(headers: row.headers, write_headers: write_headers))
|
117
118
|
if row.missing_required_columns.any?
|
118
119
|
missing_required_column_names = row.missing_required_columns.map(&:name)
|
119
|
-
@
|
120
|
+
@map.skip_or_raise(self, row, "missing required columns '#{missing_required_column_names.join(', ')}'")
|
120
121
|
else
|
121
122
|
@csv << row.serialize if append?(row.serialize)
|
122
123
|
end
|
123
|
-
@
|
124
|
+
@line += 1
|
125
|
+
end
|
126
|
+
|
127
|
+
def to_s
|
128
|
+
case @io
|
129
|
+
when File, Tempfile
|
130
|
+
@io.path
|
131
|
+
when IO
|
132
|
+
case @io.fileno
|
133
|
+
when 0
|
134
|
+
'STDIN'
|
135
|
+
when 1
|
136
|
+
'STDOUT'
|
137
|
+
when 2
|
138
|
+
'STDERR'
|
139
|
+
else
|
140
|
+
'UNKNOWN'
|
141
|
+
end
|
142
|
+
when StringIO
|
143
|
+
'StringIO'
|
144
|
+
end
|
145
|
+
end
|
146
|
+
|
147
|
+
def line
|
148
|
+
@line
|
124
149
|
end
|
125
150
|
|
126
151
|
private
|
@@ -135,11 +160,11 @@ module Masamune::Schema
|
|
135
160
|
row = Masamune::Schema::Row.new(parent: @table, values: data.to_hash, strict: false)
|
136
161
|
row.to_hash
|
137
162
|
rescue
|
138
|
-
@
|
163
|
+
@map.skip_or_raise(self, data, 'failed to parse')
|
139
164
|
end
|
140
165
|
|
141
166
|
def append?(elem)
|
142
|
-
return true unless @
|
167
|
+
return true unless @map.distinct
|
143
168
|
@seen ||= Set.new
|
144
169
|
@seen.add?(elem)
|
145
170
|
end
|
@@ -151,8 +176,9 @@ module Masamune::Schema
|
|
151
176
|
target: nil,
|
152
177
|
columns: nil,
|
153
178
|
store: nil,
|
154
|
-
function: ->(row) { row },
|
179
|
+
function: ->(row, *_) { row },
|
155
180
|
distinct: false,
|
181
|
+
fail_fast: false,
|
156
182
|
debug: false
|
157
183
|
}
|
158
184
|
|
@@ -167,6 +193,7 @@ module Masamune::Schema
|
|
167
193
|
DEFAULT_ATTRIBUTES.merge(opts).each do |name, value|
|
168
194
|
public_send("#{name}=", value)
|
169
195
|
end
|
196
|
+
@skipped = 0
|
170
197
|
end
|
171
198
|
|
172
199
|
def source=(source)
|
@@ -179,7 +206,7 @@ module Masamune::Schema
|
|
179
206
|
end
|
180
207
|
|
181
208
|
def intermediate_columns
|
182
|
-
output = function.call(default_row(source.columns))
|
209
|
+
output = function.call(default_row(source.columns), 0)
|
183
210
|
example = Array.wrap(output).first
|
184
211
|
raise ArgumentError, "function for map between '#{source.name}' and '#{target.name}' does not return output for default input" unless example
|
185
212
|
example.keys
|
@@ -190,8 +217,8 @@ module Masamune::Schema
|
|
190
217
|
end
|
191
218
|
|
192
219
|
def apply(input_files, output_file)
|
193
|
-
input_buffer = Buffer.new(source)
|
194
|
-
output_buffer = Buffer.new(
|
220
|
+
input_buffer = Buffer.new(self, source)
|
221
|
+
output_buffer = Buffer.new(self, intermediate)
|
195
222
|
self.class.convert_files(input_files).each do |input_file|
|
196
223
|
open_stream(input_file, 'r') do |input_stream|
|
197
224
|
input_buffer.bind(input_stream)
|
@@ -216,6 +243,19 @@ module Masamune::Schema
|
|
216
243
|
end
|
217
244
|
end
|
218
245
|
|
246
|
+
def skip_or_raise(buffer, row, message)
|
247
|
+
message = 'failed to process' if message.nil? || message.blank?
|
248
|
+
trace = { message: message, source: source.name, target: target.name, file: buffer.try(:to_s), line: buffer.try(:line), row: row.try(:to_hash) }
|
249
|
+
if fail_fast
|
250
|
+
@store.logger.error(message)
|
251
|
+
@store.logger.debug(trace)
|
252
|
+
raise message
|
253
|
+
else
|
254
|
+
@store.logger.warn(message)
|
255
|
+
@store.logger.debug(trace)
|
256
|
+
end
|
257
|
+
end
|
258
|
+
|
219
259
|
class << self
|
220
260
|
def convert_file(file)
|
221
261
|
if file.respond_to?(:path)
|
@@ -250,20 +290,20 @@ module Masamune::Schema
|
|
250
290
|
|
251
291
|
def apply_buffer(input_buffer, output_buffer)
|
252
292
|
input_buffer.each do |input|
|
253
|
-
safe_apply_function(input) do |output|
|
293
|
+
safe_apply_function(input_buffer, input) do |output|
|
254
294
|
output_buffer.append output
|
255
295
|
end
|
256
296
|
end
|
257
297
|
output_buffer.flush
|
258
298
|
end
|
259
299
|
|
260
|
-
def safe_apply_function(input, &block)
|
300
|
+
def safe_apply_function(input_buffer, input, &block)
|
261
301
|
return unless input
|
262
|
-
Array.wrap(function.call(input)).each do |output|
|
302
|
+
Array.wrap(function.call(input, input_buffer.line)).each do |output|
|
263
303
|
yield output
|
264
304
|
end
|
265
305
|
rescue => e
|
266
|
-
|
306
|
+
skip_or_raise(input_buffer, input, e.message)
|
267
307
|
end
|
268
308
|
end
|
269
309
|
end
|
data/lib/masamune/thor.rb
CHANGED
@@ -77,9 +77,8 @@ module Masamune
|
|
77
77
|
rescue => e
|
78
78
|
instance.logger.error("#{e.message} (#{e.class}) backtrace:")
|
79
79
|
e.backtrace.each { |x| instance.logger.error(x) }
|
80
|
-
$stderr.puts e.to_s
|
81
80
|
$stderr.puts "For complete debug log see: #{instance.log_file_name.to_s}"
|
82
|
-
|
81
|
+
abort e.message
|
83
82
|
end
|
84
83
|
end
|
85
84
|
|
data/lib/masamune/version.rb
CHANGED
@@ -71,9 +71,14 @@ describe Masamune::Actions::Transform do
|
|
71
71
|
|
72
72
|
let(:instance) { klass.new }
|
73
73
|
let(:postgres) { catalog.postgres }
|
74
|
+
let(:options) { { extra_args: true } }
|
75
|
+
|
76
|
+
before do
|
77
|
+
expect(instance).to receive(:postgres).with(hash_including(extra_args: true)).and_call_original
|
78
|
+
end
|
74
79
|
|
75
80
|
describe '.load_dimension' do
|
76
|
-
subject { instance.load_dimension(source_file, postgres.user_file, postgres.user_dimension) }
|
81
|
+
subject { instance.load_dimension(source_file, postgres.user_file, postgres.user_dimension, options) }
|
77
82
|
|
78
83
|
context 'without :map' do
|
79
84
|
before do
|
@@ -111,7 +116,7 @@ describe Masamune::Actions::Transform do
|
|
111
116
|
mock_command(/\Apsql/, mock_success)
|
112
117
|
end
|
113
118
|
|
114
|
-
subject { instance.relabel_dimension(postgres.user_dimension) }
|
119
|
+
subject { instance.relabel_dimension(postgres.user_dimension, options) }
|
115
120
|
|
116
121
|
it { is_expected.to be_success }
|
117
122
|
end
|
@@ -121,7 +126,7 @@ describe Masamune::Actions::Transform do
|
|
121
126
|
mock_command(/\Apsql/, mock_success)
|
122
127
|
end
|
123
128
|
|
124
|
-
subject { instance.consolidate_dimension(postgres.user_dimension) }
|
129
|
+
subject { instance.consolidate_dimension(postgres.user_dimension, options) }
|
125
130
|
|
126
131
|
it { is_expected.to be_success }
|
127
132
|
end
|
@@ -135,7 +140,7 @@ describe Masamune::Actions::Transform do
|
|
135
140
|
mock_command(/\Apsql/, mock_success)
|
136
141
|
end
|
137
142
|
|
138
|
-
subject { instance.load_fact(source_file, postgres.visits_hourly_file, postgres.visits_hourly_fact, date) }
|
143
|
+
subject { instance.load_fact(source_file, postgres.visits_hourly_file, postgres.visits_hourly_fact, date, options) }
|
139
144
|
|
140
145
|
it { is_expected.to be_success }
|
141
146
|
end
|
@@ -150,7 +155,7 @@ describe Masamune::Actions::Transform do
|
|
150
155
|
mock_command(/\Apsql/, mock_success)
|
151
156
|
end
|
152
157
|
|
153
|
-
subject { instance.load_fact(source_file, postgres.visits_hourly_file, postgres.visits_hourly_fact, date) }
|
158
|
+
subject { instance.load_fact(source_file, postgres.visits_hourly_file, postgres.visits_hourly_fact, date, options) }
|
154
159
|
|
155
160
|
it { is_expected.to be_success }
|
156
161
|
end
|
@@ -163,7 +168,7 @@ describe Masamune::Actions::Transform do
|
|
163
168
|
mock_command(/\Apsql/, mock_success)
|
164
169
|
end
|
165
170
|
|
166
|
-
subject { instance.rollup_fact(postgres.visits_hourly_fact, postgres.visits_daily_fact, date) }
|
171
|
+
subject { instance.rollup_fact(postgres.visits_hourly_fact, postgres.visits_daily_fact, date, options) }
|
167
172
|
|
168
173
|
it { is_expected.to be_success }
|
169
174
|
end
|
@@ -129,5 +129,28 @@ describe Masamune::Commands::Postgres do
|
|
129
129
|
end
|
130
130
|
end
|
131
131
|
|
132
|
+
describe '#failure_message' do
|
133
|
+
let(:status_code) { 1 }
|
134
|
+
|
135
|
+
context 'when error detected' do
|
136
|
+
before do
|
137
|
+
expect(instance.logger).to receive(:debug).at_least(3).times
|
138
|
+
instance.handle_stderr('Everything is OK', 0)
|
139
|
+
instance.handle_stderr('psql:/var/tmp/schema.psql ERROR: Something went wrong', 1)
|
140
|
+
instance.handle_stderr('Wha happen', 2)
|
141
|
+
end
|
142
|
+
|
143
|
+
subject { instance.failure_message(status_code) }
|
144
|
+
|
145
|
+
it { is_expected.to eq('Something went wrong') }
|
146
|
+
end
|
147
|
+
|
148
|
+
context 'when no error detected' do
|
149
|
+
subject { instance.failure_message(status_code) }
|
150
|
+
|
151
|
+
it { is_expected.to eq('psql failed without error') }
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
132
155
|
it_should_behave_like Masamune::Commands::PostgresCommon
|
133
156
|
end
|
@@ -73,6 +73,19 @@ describe Masamune::Commands::Shell do
|
|
73
73
|
it { expect { subject }.to raise_error RuntimeError, "fail_fast: #{command}" }
|
74
74
|
end
|
75
75
|
|
76
|
+
context 'with fail_fast and simple command that fails and delegate.failure_message' do
|
77
|
+
let(:command) { %Q{bash -c 'exit 1'} }
|
78
|
+
let(:options) { {fail_fast: true} }
|
79
|
+
|
80
|
+
before do
|
81
|
+
allow(delegate).to receive(:failure_message).and_return('Wha happen')
|
82
|
+
expect(instance.logger).to receive(:debug).with(%q(execute: TZ=UTC bash -c 'exit 1'))
|
83
|
+
expect(instance.logger).to receive(:debug).with(/\Astatus: .* exit 1\z/)
|
84
|
+
end
|
85
|
+
|
86
|
+
it { expect { subject }.to raise_error RuntimeError, 'Wha happen' }
|
87
|
+
end
|
88
|
+
|
76
89
|
context 'when command is interrupted' do
|
77
90
|
let(:command) { %Q{bash -c "echo 'test'"} }
|
78
91
|
|
@@ -176,7 +176,24 @@ describe Masamune::Schema::Map do
|
|
176
176
|
end
|
177
177
|
|
178
178
|
before do
|
179
|
-
expect(environment.logger).to receive(:warn).with(/
|
179
|
+
expect(environment.logger).to receive(:warn).with(/missing required columns 'user_id'/).ordered
|
180
|
+
expect(environment.logger).to receive(:debug).with(
|
181
|
+
:message => %q(missing required columns 'user_id'),
|
182
|
+
:source => 'user_stage',
|
183
|
+
:target => 'user_dimension_ledger',
|
184
|
+
:file => output.path,
|
185
|
+
:line => 3,
|
186
|
+
:row => {
|
187
|
+
'tenant_id' => 50,
|
188
|
+
'user_id' => nil,
|
189
|
+
'user_account_state.name' => 'active',
|
190
|
+
'hr_user_account_state.name' => 'active',
|
191
|
+
'admin' => false,
|
192
|
+
'preferences' => {},
|
193
|
+
'source' => 'users_file',
|
194
|
+
'cluster_id' => 100
|
195
|
+
}
|
196
|
+
).ordered
|
180
197
|
end
|
181
198
|
|
182
199
|
it 'should match target data' do
|
@@ -221,8 +238,36 @@ describe Masamune::Schema::Map do
|
|
221
238
|
end
|
222
239
|
|
223
240
|
before do
|
224
|
-
expect(environment.logger).to receive(:warn).with(/failed to process
|
225
|
-
expect(environment.logger).to receive(:
|
241
|
+
expect(environment.logger).to receive(:warn).with(/failed to process/).ordered
|
242
|
+
expect(environment.logger).to receive(:debug).with(
|
243
|
+
:message => "failed to process",
|
244
|
+
:source => "input_stage",
|
245
|
+
:target => "user_dimension_ledger",
|
246
|
+
:file => input.path,
|
247
|
+
:line => 2,
|
248
|
+
:row => {
|
249
|
+
"id" => 1,
|
250
|
+
"tenant_id" => 42,
|
251
|
+
"admin" => false,
|
252
|
+
"preferences" => {},
|
253
|
+
"deleted_at" => nil
|
254
|
+
}
|
255
|
+
).ordered
|
256
|
+
expect(environment.logger).to receive(:warn).with(/failed to parse/).ordered
|
257
|
+
expect(environment.logger).to receive(:debug).with(
|
258
|
+
:message => "failed to parse",
|
259
|
+
:source => "input_stage",
|
260
|
+
:target => "user_dimension_ledger",
|
261
|
+
:file => input.path,
|
262
|
+
:line => 4,
|
263
|
+
:row => {
|
264
|
+
:id => "3",
|
265
|
+
:tenant_id => "50",
|
266
|
+
:admin => "0",
|
267
|
+
:preferences => "INVALID_JSON",
|
268
|
+
:deleted_at => nil
|
269
|
+
}
|
270
|
+
).ordered
|
226
271
|
end
|
227
272
|
|
228
273
|
let(:target_data) do
|
@@ -495,6 +540,53 @@ describe Masamune::Schema::Map do
|
|
495
540
|
it_behaves_like 'apply input/output'
|
496
541
|
end
|
497
542
|
|
543
|
+
context 'with line_number' do
|
544
|
+
before do
|
545
|
+
catalog.schema :files do
|
546
|
+
file 'input' do
|
547
|
+
column 'id', type: :integer
|
548
|
+
end
|
549
|
+
|
550
|
+
file 'output' do
|
551
|
+
column 'id', type: :integer
|
552
|
+
column 'line_no', type: :integer
|
553
|
+
end
|
554
|
+
|
555
|
+
map from: files.input, to: files.output do |row, line_no|
|
556
|
+
{ id: row[:id], line_no: line_no }
|
557
|
+
end
|
558
|
+
end
|
559
|
+
end
|
560
|
+
|
561
|
+
let(:source) do
|
562
|
+
catalog.files.input
|
563
|
+
end
|
564
|
+
|
565
|
+
let(:target) do
|
566
|
+
catalog.files.output
|
567
|
+
end
|
568
|
+
|
569
|
+
let(:source_data) do
|
570
|
+
<<-EOS.strip_heredoc
|
571
|
+
1
|
572
|
+
2
|
573
|
+
EOS
|
574
|
+
end
|
575
|
+
|
576
|
+
let(:target_data) do
|
577
|
+
<<-EOS.strip_heredoc
|
578
|
+
1,0
|
579
|
+
2,1
|
580
|
+
EOS
|
581
|
+
end
|
582
|
+
|
583
|
+
it 'should match target data' do
|
584
|
+
is_expected.to eq(target_data)
|
585
|
+
end
|
586
|
+
|
587
|
+
it_behaves_like 'apply input/output'
|
588
|
+
end
|
589
|
+
|
498
590
|
context 'from file to table' do
|
499
591
|
before do
|
500
592
|
catalog.schema :postgres do
|
@@ -596,6 +688,112 @@ describe Masamune::Schema::Map do
|
|
596
688
|
|
597
689
|
it_behaves_like 'apply input/output'
|
598
690
|
end
|
691
|
+
|
692
|
+
context 'with map that raises exception and fail_fast' do
|
693
|
+
before do
|
694
|
+
catalog.schema :files do
|
695
|
+
file 'input' do
|
696
|
+
column 'id', type: :integer
|
697
|
+
end
|
698
|
+
|
699
|
+
file 'output' do
|
700
|
+
column 'id', type: :integer
|
701
|
+
end
|
702
|
+
|
703
|
+
map from: files.input, to: files.output, columns: [:id], fail_fast: true do |row|
|
704
|
+
raise 'wha happen'
|
705
|
+
end
|
706
|
+
end
|
707
|
+
end
|
708
|
+
|
709
|
+
let(:source) do
|
710
|
+
catalog.files.input
|
711
|
+
end
|
712
|
+
|
713
|
+
let(:target) do
|
714
|
+
catalog.files.output
|
715
|
+
end
|
716
|
+
|
717
|
+
let(:source_data) do
|
718
|
+
<<-EOS.strip_heredoc
|
719
|
+
1
|
720
|
+
2
|
721
|
+
EOS
|
722
|
+
end
|
723
|
+
|
724
|
+
let(:target_data) do
|
725
|
+
<<-EOS.strip_heredoc
|
726
|
+
1
|
727
|
+
2
|
728
|
+
EOS
|
729
|
+
end
|
730
|
+
|
731
|
+
before do
|
732
|
+
expect(environment.logger).to receive(:error).with(/wha happen/).ordered
|
733
|
+
expect(environment.logger).to receive(:debug).with(
|
734
|
+
:message => "wha happen",
|
735
|
+
:source => "input_stage",
|
736
|
+
:target => "output_stage",
|
737
|
+
:file => input.path,
|
738
|
+
:line => 0,
|
739
|
+
:row => {"id" => 1}
|
740
|
+
).ordered
|
741
|
+
end
|
742
|
+
|
743
|
+
it 'raises exception' do
|
744
|
+
expect { subject }.to raise_error /wha happen/
|
745
|
+
end
|
746
|
+
end
|
747
|
+
end
|
748
|
+
|
749
|
+
describe Masamune::Schema::Map::Buffer do
|
750
|
+
let(:map) { double }
|
751
|
+
let(:table) { double }
|
752
|
+
let(:instance) { described_class.new(map, table) }
|
753
|
+
|
754
|
+
before do
|
755
|
+
allow(table).to receive(:store).and_return(double)
|
756
|
+
instance.bind(io_delegate)
|
757
|
+
end
|
758
|
+
|
759
|
+
describe '#to_s' do
|
760
|
+
subject { instance.to_s }
|
761
|
+
|
762
|
+
context ' with STDIN' do
|
763
|
+
let(:io_delegate) { STDIN }
|
764
|
+
it { is_expected.to eq('STDIN') }
|
765
|
+
end
|
766
|
+
|
767
|
+
context ' with STDOUT' do
|
768
|
+
let(:io_delegate) { STDOUT }
|
769
|
+
it { is_expected.to eq('STDOUT') }
|
770
|
+
end
|
771
|
+
|
772
|
+
context ' with STDERR' do
|
773
|
+
let(:io_delegate) { STDERR }
|
774
|
+
it { is_expected.to eq('STDERR') }
|
775
|
+
end
|
776
|
+
|
777
|
+
context ' with unknown IO' do
|
778
|
+
let(:io_delegate) { IO.new(IO.sysopen('/dev/null', 'r')) }
|
779
|
+
it { is_expected.to eq('UNKNOWN') }
|
780
|
+
end
|
781
|
+
|
782
|
+
context ' with StringIO' do
|
783
|
+
let(:io_delegate) { StringIO.new }
|
784
|
+
it { is_expected.to eq('StringIO') }
|
785
|
+
end
|
786
|
+
|
787
|
+
context ' with File' do
|
788
|
+
let(:io_delegate) { File.new('/dev/null', 'r') }
|
789
|
+
it { is_expected.to eq(io_delegate.path) }
|
790
|
+
end
|
791
|
+
|
792
|
+
context ' with Tempfile' do
|
793
|
+
let(:io_delegate) { Tempfile.new('masamune') }
|
794
|
+
it { is_expected.to eq(io_delegate.path) }
|
795
|
+
end
|
796
|
+
end
|
599
797
|
end
|
600
798
|
|
601
799
|
describe Masamune::Schema::Map::JSONEncoder do
|
data/spec/masamune/thor_spec.rb
CHANGED
@@ -101,6 +101,7 @@ describe Masamune::Thor do
|
|
101
101
|
it 'exits with status code 0 and prints version' do
|
102
102
|
expect { cli_invocation }.to raise_error { |e|
|
103
103
|
expect(e).to be_a(SystemExit)
|
104
|
+
expect(e.message).to eq('exit')
|
104
105
|
expect(e.status).to eq(0)
|
105
106
|
}
|
106
107
|
expect(stdout.string).to match(/\Amasamune/)
|
@@ -206,13 +207,13 @@ describe Masamune::Thor do
|
|
206
207
|
it 'exits with status code 1 and prints error to stderr' do
|
207
208
|
expect { cli_invocation }.to raise_error { |e|
|
208
209
|
expect(e).to be_a(SystemExit)
|
210
|
+
expect(e.message).to eq('Path :unknown_dir not defined')
|
209
211
|
expect(e.status).to eq(1)
|
210
212
|
}
|
211
213
|
expect(stdout.string).to be_blank
|
212
214
|
expect(stderr.string).to match(/Path :unknown_dir not defined/)
|
213
215
|
end
|
214
216
|
end
|
215
|
-
|
216
217
|
end
|
217
218
|
|
218
219
|
context 'with command that prints :current_dir' do
|
metadata
CHANGED
@@ -1,195 +1,195 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: masamune
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.17.
|
4
|
+
version: 0.17.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Michael Andrews
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-02-
|
11
|
+
date: 2016-02-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: thor
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- -
|
17
|
+
- - '>='
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '0'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- -
|
24
|
+
- - '>='
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: activesupport
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- -
|
31
|
+
- - '>='
|
32
32
|
- !ruby/object:Gem::Version
|
33
33
|
version: '0'
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- -
|
38
|
+
- - '>='
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: tzinfo
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- -
|
45
|
+
- - '>='
|
46
46
|
- !ruby/object:Gem::Version
|
47
47
|
version: '0'
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- -
|
52
|
+
- - '>='
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '0'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: chronic
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- -
|
59
|
+
- - '>='
|
60
60
|
- !ruby/object:Gem::Version
|
61
61
|
version: '0'
|
62
62
|
type: :runtime
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
|
-
- -
|
66
|
+
- - '>='
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: tilt
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
|
-
- -
|
73
|
+
- - '>='
|
74
74
|
- !ruby/object:Gem::Version
|
75
75
|
version: '0'
|
76
76
|
type: :runtime
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
|
-
- -
|
80
|
+
- - '>='
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: '0'
|
83
83
|
- !ruby/object:Gem::Dependency
|
84
84
|
name: erubis
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
86
86
|
requirements:
|
87
|
-
- -
|
87
|
+
- - '>='
|
88
88
|
- !ruby/object:Gem::Version
|
89
89
|
version: '0'
|
90
90
|
type: :runtime
|
91
91
|
prerelease: false
|
92
92
|
version_requirements: !ruby/object:Gem::Requirement
|
93
93
|
requirements:
|
94
|
-
- -
|
94
|
+
- - '>='
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: '0'
|
97
97
|
- !ruby/object:Gem::Dependency
|
98
98
|
name: parallel
|
99
99
|
requirement: !ruby/object:Gem::Requirement
|
100
100
|
requirements:
|
101
|
-
- -
|
101
|
+
- - '>='
|
102
102
|
- !ruby/object:Gem::Version
|
103
103
|
version: '0'
|
104
104
|
type: :runtime
|
105
105
|
prerelease: false
|
106
106
|
version_requirements: !ruby/object:Gem::Requirement
|
107
107
|
requirements:
|
108
|
-
- -
|
108
|
+
- - '>='
|
109
109
|
- !ruby/object:Gem::Version
|
110
110
|
version: '0'
|
111
111
|
- !ruby/object:Gem::Dependency
|
112
112
|
name: pry
|
113
113
|
requirement: !ruby/object:Gem::Requirement
|
114
114
|
requirements:
|
115
|
-
- -
|
115
|
+
- - '>='
|
116
116
|
- !ruby/object:Gem::Version
|
117
117
|
version: '0'
|
118
118
|
type: :runtime
|
119
119
|
prerelease: false
|
120
120
|
version_requirements: !ruby/object:Gem::Requirement
|
121
121
|
requirements:
|
122
|
-
- -
|
122
|
+
- - '>='
|
123
123
|
- !ruby/object:Gem::Version
|
124
124
|
version: '0'
|
125
125
|
- !ruby/object:Gem::Dependency
|
126
126
|
name: rb-readline
|
127
127
|
requirement: !ruby/object:Gem::Requirement
|
128
128
|
requirements:
|
129
|
-
- -
|
129
|
+
- - '>='
|
130
130
|
- !ruby/object:Gem::Version
|
131
131
|
version: '0'
|
132
132
|
type: :runtime
|
133
133
|
prerelease: false
|
134
134
|
version_requirements: !ruby/object:Gem::Requirement
|
135
135
|
requirements:
|
136
|
-
- -
|
136
|
+
- - '>='
|
137
137
|
- !ruby/object:Gem::Version
|
138
138
|
version: '0'
|
139
139
|
- !ruby/object:Gem::Dependency
|
140
140
|
name: rake
|
141
141
|
requirement: !ruby/object:Gem::Requirement
|
142
142
|
requirements:
|
143
|
-
- -
|
143
|
+
- - '>='
|
144
144
|
- !ruby/object:Gem::Version
|
145
145
|
version: '0.9'
|
146
146
|
type: :development
|
147
147
|
prerelease: false
|
148
148
|
version_requirements: !ruby/object:Gem::Requirement
|
149
149
|
requirements:
|
150
|
-
- -
|
150
|
+
- - '>='
|
151
151
|
- !ruby/object:Gem::Version
|
152
152
|
version: '0.9'
|
153
153
|
- !ruby/object:Gem::Dependency
|
154
154
|
name: user_agent_parser
|
155
155
|
requirement: !ruby/object:Gem::Requirement
|
156
156
|
requirements:
|
157
|
-
- -
|
157
|
+
- - '>='
|
158
158
|
- !ruby/object:Gem::Version
|
159
159
|
version: '0'
|
160
160
|
type: :development
|
161
161
|
prerelease: false
|
162
162
|
version_requirements: !ruby/object:Gem::Requirement
|
163
163
|
requirements:
|
164
|
-
- -
|
164
|
+
- - '>='
|
165
165
|
- !ruby/object:Gem::Version
|
166
166
|
version: '0'
|
167
167
|
- !ruby/object:Gem::Dependency
|
168
168
|
name: rspec
|
169
169
|
requirement: !ruby/object:Gem::Requirement
|
170
170
|
requirements:
|
171
|
-
- -
|
171
|
+
- - '>'
|
172
172
|
- !ruby/object:Gem::Version
|
173
173
|
version: '2.99'
|
174
174
|
type: :development
|
175
175
|
prerelease: false
|
176
176
|
version_requirements: !ruby/object:Gem::Requirement
|
177
177
|
requirements:
|
178
|
-
- -
|
178
|
+
- - '>'
|
179
179
|
- !ruby/object:Gem::Version
|
180
180
|
version: '2.99'
|
181
181
|
- !ruby/object:Gem::Dependency
|
182
182
|
name: byebug
|
183
183
|
requirement: !ruby/object:Gem::Requirement
|
184
184
|
requirements:
|
185
|
-
- -
|
185
|
+
- - '>='
|
186
186
|
- !ruby/object:Gem::Version
|
187
187
|
version: '0'
|
188
188
|
type: :development
|
189
189
|
prerelease: false
|
190
190
|
version_requirements: !ruby/object:Gem::Requirement
|
191
191
|
requirements:
|
192
|
-
- -
|
192
|
+
- - '>='
|
193
193
|
- !ruby/object:Gem::Version
|
194
194
|
version: '0'
|
195
195
|
description: Hybrid Data & Work Flow
|
@@ -426,17 +426,17 @@ require_paths:
|
|
426
426
|
- lib
|
427
427
|
required_ruby_version: !ruby/object:Gem::Requirement
|
428
428
|
requirements:
|
429
|
-
- -
|
429
|
+
- - '>='
|
430
430
|
- !ruby/object:Gem::Version
|
431
431
|
version: '0'
|
432
432
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
433
433
|
requirements:
|
434
|
-
- -
|
434
|
+
- - '>='
|
435
435
|
- !ruby/object:Gem::Version
|
436
436
|
version: '0'
|
437
437
|
requirements: []
|
438
438
|
rubyforge_project:
|
439
|
-
rubygems_version: 2.4.
|
439
|
+
rubygems_version: 2.4.8
|
440
440
|
signing_key:
|
441
441
|
specification_version: 4
|
442
442
|
summary: Hybrid Data & Work Flow
|