masamune 0.17.5 → 0.17.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/masamune/actions/transform.rb +10 -10
- data/lib/masamune/commands/postgres.rb +10 -0
- data/lib/masamune/commands/shell.rb +9 -1
- data/lib/masamune/schema/map.rb +56 -16
- data/lib/masamune/thor.rb +1 -2
- data/lib/masamune/version.rb +1 -1
- data/spec/masamune/actions/transform_spec.rb +11 -6
- data/spec/masamune/commands/postgres_spec.rb +23 -0
- data/spec/masamune/commands/shell_spec.rb +13 -0
- data/spec/masamune/schema/map_spec.rb +201 -3
- data/spec/masamune/thor_spec.rb +2 -1
- data/spec/support/masamune/shared_example_group.rb +0 -2
- data/spec/support/masamune/task_example_group.rb +0 -1
- metadata +31 -31
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c29146a9633b2cf7760df9db790297c0889d856c
|
4
|
+
data.tar.gz: d655ab1bd2088a9535b132dba044fe94a9c6eaa8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: bd15f96322e460a7e01c88465d4b245ee27242729c06b0567ad788fd5d02c57a72aea02b5e7cf58d9fe5217018e33b271f6723c8e522fcea37fbbb95b56afbbd
|
7
|
+
data.tar.gz: 79c5a0895b9a64ff4a7b0ce388fc344a05e21b10c674b1069fe671ce486e55e2eaddb8e81792f513da95dec966f4e9380b537627faa93e75f19828ebf90df414
|
@@ -46,33 +46,33 @@ module Masamune::Actions
|
|
46
46
|
|
47
47
|
FILE_MODE = 0777 - File.umask
|
48
48
|
|
49
|
-
def load_dimension(source_files, source, target)
|
49
|
+
def load_dimension(source_files, source, target, options = {})
|
50
50
|
optional_apply_map(source_files, source, target) do |intermediate_files, intermediate|
|
51
51
|
transform = Wrapper.load_dimension(intermediate_files, intermediate, target)
|
52
|
-
postgres file: transform.to_file, debug: (source.debug || target.debug || intermediate.debug)
|
52
|
+
postgres file: transform.to_file, debug: (source.debug || target.debug || intermediate.debug), **options
|
53
53
|
end
|
54
54
|
end
|
55
55
|
|
56
|
-
def consolidate_dimension(target)
|
56
|
+
def consolidate_dimension(target, options = {})
|
57
57
|
transform = Wrapper.consolidate_dimension(target)
|
58
|
-
postgres file: transform.to_file, debug: target.debug
|
58
|
+
postgres file: transform.to_file, debug: target.debug, **options
|
59
59
|
end
|
60
60
|
|
61
|
-
def relabel_dimension(target)
|
61
|
+
def relabel_dimension(target, options = {})
|
62
62
|
transform = Wrapper.relabel_dimension(target)
|
63
|
-
postgres file: transform.to_file, debug: target.debug
|
63
|
+
postgres file: transform.to_file, debug: target.debug, **options
|
64
64
|
end
|
65
65
|
|
66
|
-
def load_fact(source_files, source, target, date)
|
66
|
+
def load_fact(source_files, source, target, date, options = {})
|
67
67
|
optional_apply_map(source_files, source, target) do |intermediate_files, intermediate|
|
68
68
|
transform = Wrapper.load_fact(intermediate_files, intermediate, target, date)
|
69
|
-
postgres file: transform.to_file, debug: (source.debug || target.debug || intermediate.debug)
|
69
|
+
postgres file: transform.to_file, debug: (source.debug || target.debug || intermediate.debug), **options
|
70
70
|
end
|
71
71
|
end
|
72
72
|
|
73
|
-
def rollup_fact(source, target, date)
|
73
|
+
def rollup_fact(source, target, date, options = {})
|
74
74
|
transform = Wrapper.rollup_fact(source, target, date)
|
75
|
-
postgres file: transform.to_file, debug: (source.debug || target.debug)
|
75
|
+
postgres file: transform.to_file, debug: (source.debug || target.debug), **options
|
76
76
|
end
|
77
77
|
|
78
78
|
private
|
@@ -56,6 +56,7 @@ module Masamune::Commands
|
|
56
56
|
instance_variable_set("@#{name}", value)
|
57
57
|
end
|
58
58
|
raise ArgumentError, 'Cannot specify both file and exec' if @file && @exec
|
59
|
+
@error = nil
|
59
60
|
end
|
60
61
|
|
61
62
|
def stdin
|
@@ -109,6 +110,15 @@ module Masamune::Commands
|
|
109
110
|
end
|
110
111
|
end
|
111
112
|
|
113
|
+
def handle_stderr(line, line_no)
|
114
|
+
@error = line.split(/ERROR:\s*/).last if line =~ /ERROR:/
|
115
|
+
logger.debug(line)
|
116
|
+
end
|
117
|
+
|
118
|
+
def failure_message(status)
|
119
|
+
@error || 'psql failed without error'
|
120
|
+
end
|
121
|
+
|
112
122
|
def prompt
|
113
123
|
@database + '=>'
|
114
124
|
end
|
@@ -189,7 +189,15 @@ module Masamune::Commands
|
|
189
189
|
if @delegate.respond_to?(:handle_failure)
|
190
190
|
@delegate.handle_failure(status)
|
191
191
|
end
|
192
|
-
raise
|
192
|
+
raise failure_message(status) if fail_fast
|
193
|
+
end
|
194
|
+
|
195
|
+
def failure_message(status)
|
196
|
+
if @delegate.respond_to?(:failure_message)
|
197
|
+
@delegate.failure_message(status)
|
198
|
+
else
|
199
|
+
"fail_fast: #{command_args.join(' ')}"
|
200
|
+
end
|
193
201
|
end
|
194
202
|
|
195
203
|
private
|
data/lib/masamune/schema/map.rb
CHANGED
@@ -89,11 +89,11 @@ module Masamune::Schema
|
|
89
89
|
|
90
90
|
def_delegators :@io, :flush, :path
|
91
91
|
|
92
|
-
def initialize(
|
92
|
+
def initialize(map, table)
|
93
|
+
@map = map
|
93
94
|
@table = table
|
94
95
|
@store = table.store
|
95
|
-
@
|
96
|
-
@options = options
|
96
|
+
@line = 0
|
97
97
|
end
|
98
98
|
|
99
99
|
def bind(io)
|
@@ -106,21 +106,46 @@ module Masamune::Schema
|
|
106
106
|
CSV.parse(JSONEncoder.new(@io, @store), options.merge(headers: @store.headers || @table.columns.keys)) do |data|
|
107
107
|
next if data.to_s =~ /\A#/
|
108
108
|
yield safe_row(data)
|
109
|
+
@line += 1
|
109
110
|
end
|
110
111
|
end
|
111
112
|
|
112
113
|
def append(data)
|
113
114
|
raise 'must call Buffer#bind first' unless @io
|
114
115
|
row = Masamune::Schema::Row.new(parent: @table, values: data.to_hash)
|
115
|
-
write_headers = @store.headers && @
|
116
|
+
write_headers = @store.headers && @line < 1
|
116
117
|
@csv ||= CSV.new(@io, options.merge(headers: row.headers, write_headers: write_headers))
|
117
118
|
if row.missing_required_columns.any?
|
118
119
|
missing_required_column_names = row.missing_required_columns.map(&:name)
|
119
|
-
@
|
120
|
+
@map.skip_or_raise(self, row, "missing required columns '#{missing_required_column_names.join(', ')}'")
|
120
121
|
else
|
121
122
|
@csv << row.serialize if append?(row.serialize)
|
122
123
|
end
|
123
|
-
@
|
124
|
+
@line += 1
|
125
|
+
end
|
126
|
+
|
127
|
+
def to_s
|
128
|
+
case @io
|
129
|
+
when File, Tempfile
|
130
|
+
@io.path
|
131
|
+
when IO
|
132
|
+
case @io.fileno
|
133
|
+
when 0
|
134
|
+
'STDIN'
|
135
|
+
when 1
|
136
|
+
'STDOUT'
|
137
|
+
when 2
|
138
|
+
'STDERR'
|
139
|
+
else
|
140
|
+
'UNKNOWN'
|
141
|
+
end
|
142
|
+
when StringIO
|
143
|
+
'StringIO'
|
144
|
+
end
|
145
|
+
end
|
146
|
+
|
147
|
+
def line
|
148
|
+
@line
|
124
149
|
end
|
125
150
|
|
126
151
|
private
|
@@ -135,11 +160,11 @@ module Masamune::Schema
|
|
135
160
|
row = Masamune::Schema::Row.new(parent: @table, values: data.to_hash, strict: false)
|
136
161
|
row.to_hash
|
137
162
|
rescue
|
138
|
-
@
|
163
|
+
@map.skip_or_raise(self, data, 'failed to parse')
|
139
164
|
end
|
140
165
|
|
141
166
|
def append?(elem)
|
142
|
-
return true unless @
|
167
|
+
return true unless @map.distinct
|
143
168
|
@seen ||= Set.new
|
144
169
|
@seen.add?(elem)
|
145
170
|
end
|
@@ -151,8 +176,9 @@ module Masamune::Schema
|
|
151
176
|
target: nil,
|
152
177
|
columns: nil,
|
153
178
|
store: nil,
|
154
|
-
function: ->(row) { row },
|
179
|
+
function: ->(row, *_) { row },
|
155
180
|
distinct: false,
|
181
|
+
fail_fast: false,
|
156
182
|
debug: false
|
157
183
|
}
|
158
184
|
|
@@ -167,6 +193,7 @@ module Masamune::Schema
|
|
167
193
|
DEFAULT_ATTRIBUTES.merge(opts).each do |name, value|
|
168
194
|
public_send("#{name}=", value)
|
169
195
|
end
|
196
|
+
@skipped = 0
|
170
197
|
end
|
171
198
|
|
172
199
|
def source=(source)
|
@@ -179,7 +206,7 @@ module Masamune::Schema
|
|
179
206
|
end
|
180
207
|
|
181
208
|
def intermediate_columns
|
182
|
-
output = function.call(default_row(source.columns))
|
209
|
+
output = function.call(default_row(source.columns), 0)
|
183
210
|
example = Array.wrap(output).first
|
184
211
|
raise ArgumentError, "function for map between '#{source.name}' and '#{target.name}' does not return output for default input" unless example
|
185
212
|
example.keys
|
@@ -190,8 +217,8 @@ module Masamune::Schema
|
|
190
217
|
end
|
191
218
|
|
192
219
|
def apply(input_files, output_file)
|
193
|
-
input_buffer = Buffer.new(source)
|
194
|
-
output_buffer = Buffer.new(
|
220
|
+
input_buffer = Buffer.new(self, source)
|
221
|
+
output_buffer = Buffer.new(self, intermediate)
|
195
222
|
self.class.convert_files(input_files).each do |input_file|
|
196
223
|
open_stream(input_file, 'r') do |input_stream|
|
197
224
|
input_buffer.bind(input_stream)
|
@@ -216,6 +243,19 @@ module Masamune::Schema
|
|
216
243
|
end
|
217
244
|
end
|
218
245
|
|
246
|
+
def skip_or_raise(buffer, row, message)
|
247
|
+
message = 'failed to process' if message.nil? || message.blank?
|
248
|
+
trace = { message: message, source: source.name, target: target.name, file: buffer.try(:to_s), line: buffer.try(:line), row: row.try(:to_hash) }
|
249
|
+
if fail_fast
|
250
|
+
@store.logger.error(message)
|
251
|
+
@store.logger.debug(trace)
|
252
|
+
raise message
|
253
|
+
else
|
254
|
+
@store.logger.warn(message)
|
255
|
+
@store.logger.debug(trace)
|
256
|
+
end
|
257
|
+
end
|
258
|
+
|
219
259
|
class << self
|
220
260
|
def convert_file(file)
|
221
261
|
if file.respond_to?(:path)
|
@@ -250,20 +290,20 @@ module Masamune::Schema
|
|
250
290
|
|
251
291
|
def apply_buffer(input_buffer, output_buffer)
|
252
292
|
input_buffer.each do |input|
|
253
|
-
safe_apply_function(input) do |output|
|
293
|
+
safe_apply_function(input_buffer, input) do |output|
|
254
294
|
output_buffer.append output
|
255
295
|
end
|
256
296
|
end
|
257
297
|
output_buffer.flush
|
258
298
|
end
|
259
299
|
|
260
|
-
def safe_apply_function(input, &block)
|
300
|
+
def safe_apply_function(input_buffer, input, &block)
|
261
301
|
return unless input
|
262
|
-
Array.wrap(function.call(input)).each do |output|
|
302
|
+
Array.wrap(function.call(input, input_buffer.line)).each do |output|
|
263
303
|
yield output
|
264
304
|
end
|
265
305
|
rescue => e
|
266
|
-
|
306
|
+
skip_or_raise(input_buffer, input, e.message)
|
267
307
|
end
|
268
308
|
end
|
269
309
|
end
|
data/lib/masamune/thor.rb
CHANGED
@@ -77,9 +77,8 @@ module Masamune
|
|
77
77
|
rescue => e
|
78
78
|
instance.logger.error("#{e.message} (#{e.class}) backtrace:")
|
79
79
|
e.backtrace.each { |x| instance.logger.error(x) }
|
80
|
-
$stderr.puts e.to_s
|
81
80
|
$stderr.puts "For complete debug log see: #{instance.log_file_name.to_s}"
|
82
|
-
|
81
|
+
abort e.message
|
83
82
|
end
|
84
83
|
end
|
85
84
|
|
data/lib/masamune/version.rb
CHANGED
@@ -71,9 +71,14 @@ describe Masamune::Actions::Transform do
|
|
71
71
|
|
72
72
|
let(:instance) { klass.new }
|
73
73
|
let(:postgres) { catalog.postgres }
|
74
|
+
let(:options) { { extra_args: true } }
|
75
|
+
|
76
|
+
before do
|
77
|
+
expect(instance).to receive(:postgres).with(hash_including(extra_args: true)).and_call_original
|
78
|
+
end
|
74
79
|
|
75
80
|
describe '.load_dimension' do
|
76
|
-
subject { instance.load_dimension(source_file, postgres.user_file, postgres.user_dimension) }
|
81
|
+
subject { instance.load_dimension(source_file, postgres.user_file, postgres.user_dimension, options) }
|
77
82
|
|
78
83
|
context 'without :map' do
|
79
84
|
before do
|
@@ -111,7 +116,7 @@ describe Masamune::Actions::Transform do
|
|
111
116
|
mock_command(/\Apsql/, mock_success)
|
112
117
|
end
|
113
118
|
|
114
|
-
subject { instance.relabel_dimension(postgres.user_dimension) }
|
119
|
+
subject { instance.relabel_dimension(postgres.user_dimension, options) }
|
115
120
|
|
116
121
|
it { is_expected.to be_success }
|
117
122
|
end
|
@@ -121,7 +126,7 @@ describe Masamune::Actions::Transform do
|
|
121
126
|
mock_command(/\Apsql/, mock_success)
|
122
127
|
end
|
123
128
|
|
124
|
-
subject { instance.consolidate_dimension(postgres.user_dimension) }
|
129
|
+
subject { instance.consolidate_dimension(postgres.user_dimension, options) }
|
125
130
|
|
126
131
|
it { is_expected.to be_success }
|
127
132
|
end
|
@@ -135,7 +140,7 @@ describe Masamune::Actions::Transform do
|
|
135
140
|
mock_command(/\Apsql/, mock_success)
|
136
141
|
end
|
137
142
|
|
138
|
-
subject { instance.load_fact(source_file, postgres.visits_hourly_file, postgres.visits_hourly_fact, date) }
|
143
|
+
subject { instance.load_fact(source_file, postgres.visits_hourly_file, postgres.visits_hourly_fact, date, options) }
|
139
144
|
|
140
145
|
it { is_expected.to be_success }
|
141
146
|
end
|
@@ -150,7 +155,7 @@ describe Masamune::Actions::Transform do
|
|
150
155
|
mock_command(/\Apsql/, mock_success)
|
151
156
|
end
|
152
157
|
|
153
|
-
subject { instance.load_fact(source_file, postgres.visits_hourly_file, postgres.visits_hourly_fact, date) }
|
158
|
+
subject { instance.load_fact(source_file, postgres.visits_hourly_file, postgres.visits_hourly_fact, date, options) }
|
154
159
|
|
155
160
|
it { is_expected.to be_success }
|
156
161
|
end
|
@@ -163,7 +168,7 @@ describe Masamune::Actions::Transform do
|
|
163
168
|
mock_command(/\Apsql/, mock_success)
|
164
169
|
end
|
165
170
|
|
166
|
-
subject { instance.rollup_fact(postgres.visits_hourly_fact, postgres.visits_daily_fact, date) }
|
171
|
+
subject { instance.rollup_fact(postgres.visits_hourly_fact, postgres.visits_daily_fact, date, options) }
|
167
172
|
|
168
173
|
it { is_expected.to be_success }
|
169
174
|
end
|
@@ -129,5 +129,28 @@ describe Masamune::Commands::Postgres do
|
|
129
129
|
end
|
130
130
|
end
|
131
131
|
|
132
|
+
describe '#failure_message' do
|
133
|
+
let(:status_code) { 1 }
|
134
|
+
|
135
|
+
context 'when error detected' do
|
136
|
+
before do
|
137
|
+
expect(instance.logger).to receive(:debug).at_least(3).times
|
138
|
+
instance.handle_stderr('Everything is OK', 0)
|
139
|
+
instance.handle_stderr('psql:/var/tmp/schema.psql ERROR: Something went wrong', 1)
|
140
|
+
instance.handle_stderr('Wha happen', 2)
|
141
|
+
end
|
142
|
+
|
143
|
+
subject { instance.failure_message(status_code) }
|
144
|
+
|
145
|
+
it { is_expected.to eq('Something went wrong') }
|
146
|
+
end
|
147
|
+
|
148
|
+
context 'when no error detected' do
|
149
|
+
subject { instance.failure_message(status_code) }
|
150
|
+
|
151
|
+
it { is_expected.to eq('psql failed without error') }
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
132
155
|
it_should_behave_like Masamune::Commands::PostgresCommon
|
133
156
|
end
|
@@ -73,6 +73,19 @@ describe Masamune::Commands::Shell do
|
|
73
73
|
it { expect { subject }.to raise_error RuntimeError, "fail_fast: #{command}" }
|
74
74
|
end
|
75
75
|
|
76
|
+
context 'with fail_fast and simple command that fails and delegate.failure_message' do
|
77
|
+
let(:command) { %Q{bash -c 'exit 1'} }
|
78
|
+
let(:options) { {fail_fast: true} }
|
79
|
+
|
80
|
+
before do
|
81
|
+
allow(delegate).to receive(:failure_message).and_return('Wha happen')
|
82
|
+
expect(instance.logger).to receive(:debug).with(%q(execute: TZ=UTC bash -c 'exit 1'))
|
83
|
+
expect(instance.logger).to receive(:debug).with(/\Astatus: .* exit 1\z/)
|
84
|
+
end
|
85
|
+
|
86
|
+
it { expect { subject }.to raise_error RuntimeError, 'Wha happen' }
|
87
|
+
end
|
88
|
+
|
76
89
|
context 'when command is interrupted' do
|
77
90
|
let(:command) { %Q{bash -c "echo 'test'"} }
|
78
91
|
|
@@ -176,7 +176,24 @@ describe Masamune::Schema::Map do
|
|
176
176
|
end
|
177
177
|
|
178
178
|
before do
|
179
|
-
expect(environment.logger).to receive(:warn).with(/
|
179
|
+
expect(environment.logger).to receive(:warn).with(/missing required columns 'user_id'/).ordered
|
180
|
+
expect(environment.logger).to receive(:debug).with(
|
181
|
+
:message => %q(missing required columns 'user_id'),
|
182
|
+
:source => 'user_stage',
|
183
|
+
:target => 'user_dimension_ledger',
|
184
|
+
:file => output.path,
|
185
|
+
:line => 3,
|
186
|
+
:row => {
|
187
|
+
'tenant_id' => 50,
|
188
|
+
'user_id' => nil,
|
189
|
+
'user_account_state.name' => 'active',
|
190
|
+
'hr_user_account_state.name' => 'active',
|
191
|
+
'admin' => false,
|
192
|
+
'preferences' => {},
|
193
|
+
'source' => 'users_file',
|
194
|
+
'cluster_id' => 100
|
195
|
+
}
|
196
|
+
).ordered
|
180
197
|
end
|
181
198
|
|
182
199
|
it 'should match target data' do
|
@@ -221,8 +238,36 @@ describe Masamune::Schema::Map do
|
|
221
238
|
end
|
222
239
|
|
223
240
|
before do
|
224
|
-
expect(environment.logger).to receive(:warn).with(/failed to process
|
225
|
-
expect(environment.logger).to receive(:
|
241
|
+
expect(environment.logger).to receive(:warn).with(/failed to process/).ordered
|
242
|
+
expect(environment.logger).to receive(:debug).with(
|
243
|
+
:message => "failed to process",
|
244
|
+
:source => "input_stage",
|
245
|
+
:target => "user_dimension_ledger",
|
246
|
+
:file => input.path,
|
247
|
+
:line => 2,
|
248
|
+
:row => {
|
249
|
+
"id" => 1,
|
250
|
+
"tenant_id" => 42,
|
251
|
+
"admin" => false,
|
252
|
+
"preferences" => {},
|
253
|
+
"deleted_at" => nil
|
254
|
+
}
|
255
|
+
).ordered
|
256
|
+
expect(environment.logger).to receive(:warn).with(/failed to parse/).ordered
|
257
|
+
expect(environment.logger).to receive(:debug).with(
|
258
|
+
:message => "failed to parse",
|
259
|
+
:source => "input_stage",
|
260
|
+
:target => "user_dimension_ledger",
|
261
|
+
:file => input.path,
|
262
|
+
:line => 4,
|
263
|
+
:row => {
|
264
|
+
:id => "3",
|
265
|
+
:tenant_id => "50",
|
266
|
+
:admin => "0",
|
267
|
+
:preferences => "INVALID_JSON",
|
268
|
+
:deleted_at => nil
|
269
|
+
}
|
270
|
+
).ordered
|
226
271
|
end
|
227
272
|
|
228
273
|
let(:target_data) do
|
@@ -495,6 +540,53 @@ describe Masamune::Schema::Map do
|
|
495
540
|
it_behaves_like 'apply input/output'
|
496
541
|
end
|
497
542
|
|
543
|
+
context 'with line_number' do
|
544
|
+
before do
|
545
|
+
catalog.schema :files do
|
546
|
+
file 'input' do
|
547
|
+
column 'id', type: :integer
|
548
|
+
end
|
549
|
+
|
550
|
+
file 'output' do
|
551
|
+
column 'id', type: :integer
|
552
|
+
column 'line_no', type: :integer
|
553
|
+
end
|
554
|
+
|
555
|
+
map from: files.input, to: files.output do |row, line_no|
|
556
|
+
{ id: row[:id], line_no: line_no }
|
557
|
+
end
|
558
|
+
end
|
559
|
+
end
|
560
|
+
|
561
|
+
let(:source) do
|
562
|
+
catalog.files.input
|
563
|
+
end
|
564
|
+
|
565
|
+
let(:target) do
|
566
|
+
catalog.files.output
|
567
|
+
end
|
568
|
+
|
569
|
+
let(:source_data) do
|
570
|
+
<<-EOS.strip_heredoc
|
571
|
+
1
|
572
|
+
2
|
573
|
+
EOS
|
574
|
+
end
|
575
|
+
|
576
|
+
let(:target_data) do
|
577
|
+
<<-EOS.strip_heredoc
|
578
|
+
1,0
|
579
|
+
2,1
|
580
|
+
EOS
|
581
|
+
end
|
582
|
+
|
583
|
+
it 'should match target data' do
|
584
|
+
is_expected.to eq(target_data)
|
585
|
+
end
|
586
|
+
|
587
|
+
it_behaves_like 'apply input/output'
|
588
|
+
end
|
589
|
+
|
498
590
|
context 'from file to table' do
|
499
591
|
before do
|
500
592
|
catalog.schema :postgres do
|
@@ -596,6 +688,112 @@ describe Masamune::Schema::Map do
|
|
596
688
|
|
597
689
|
it_behaves_like 'apply input/output'
|
598
690
|
end
|
691
|
+
|
692
|
+
context 'with map that raises exception and fail_fast' do
|
693
|
+
before do
|
694
|
+
catalog.schema :files do
|
695
|
+
file 'input' do
|
696
|
+
column 'id', type: :integer
|
697
|
+
end
|
698
|
+
|
699
|
+
file 'output' do
|
700
|
+
column 'id', type: :integer
|
701
|
+
end
|
702
|
+
|
703
|
+
map from: files.input, to: files.output, columns: [:id], fail_fast: true do |row|
|
704
|
+
raise 'wha happen'
|
705
|
+
end
|
706
|
+
end
|
707
|
+
end
|
708
|
+
|
709
|
+
let(:source) do
|
710
|
+
catalog.files.input
|
711
|
+
end
|
712
|
+
|
713
|
+
let(:target) do
|
714
|
+
catalog.files.output
|
715
|
+
end
|
716
|
+
|
717
|
+
let(:source_data) do
|
718
|
+
<<-EOS.strip_heredoc
|
719
|
+
1
|
720
|
+
2
|
721
|
+
EOS
|
722
|
+
end
|
723
|
+
|
724
|
+
let(:target_data) do
|
725
|
+
<<-EOS.strip_heredoc
|
726
|
+
1
|
727
|
+
2
|
728
|
+
EOS
|
729
|
+
end
|
730
|
+
|
731
|
+
before do
|
732
|
+
expect(environment.logger).to receive(:error).with(/wha happen/).ordered
|
733
|
+
expect(environment.logger).to receive(:debug).with(
|
734
|
+
:message => "wha happen",
|
735
|
+
:source => "input_stage",
|
736
|
+
:target => "output_stage",
|
737
|
+
:file => input.path,
|
738
|
+
:line => 0,
|
739
|
+
:row => {"id" => 1}
|
740
|
+
).ordered
|
741
|
+
end
|
742
|
+
|
743
|
+
it 'raises exception' do
|
744
|
+
expect { subject }.to raise_error /wha happen/
|
745
|
+
end
|
746
|
+
end
|
747
|
+
end
|
748
|
+
|
749
|
+
describe Masamune::Schema::Map::Buffer do
|
750
|
+
let(:map) { double }
|
751
|
+
let(:table) { double }
|
752
|
+
let(:instance) { described_class.new(map, table) }
|
753
|
+
|
754
|
+
before do
|
755
|
+
allow(table).to receive(:store).and_return(double)
|
756
|
+
instance.bind(io_delegate)
|
757
|
+
end
|
758
|
+
|
759
|
+
describe '#to_s' do
|
760
|
+
subject { instance.to_s }
|
761
|
+
|
762
|
+
context ' with STDIN' do
|
763
|
+
let(:io_delegate) { STDIN }
|
764
|
+
it { is_expected.to eq('STDIN') }
|
765
|
+
end
|
766
|
+
|
767
|
+
context ' with STDOUT' do
|
768
|
+
let(:io_delegate) { STDOUT }
|
769
|
+
it { is_expected.to eq('STDOUT') }
|
770
|
+
end
|
771
|
+
|
772
|
+
context ' with STDERR' do
|
773
|
+
let(:io_delegate) { STDERR }
|
774
|
+
it { is_expected.to eq('STDERR') }
|
775
|
+
end
|
776
|
+
|
777
|
+
context ' with unknown IO' do
|
778
|
+
let(:io_delegate) { IO.new(IO.sysopen('/dev/null', 'r')) }
|
779
|
+
it { is_expected.to eq('UNKNOWN') }
|
780
|
+
end
|
781
|
+
|
782
|
+
context ' with StringIO' do
|
783
|
+
let(:io_delegate) { StringIO.new }
|
784
|
+
it { is_expected.to eq('StringIO') }
|
785
|
+
end
|
786
|
+
|
787
|
+
context ' with File' do
|
788
|
+
let(:io_delegate) { File.new('/dev/null', 'r') }
|
789
|
+
it { is_expected.to eq(io_delegate.path) }
|
790
|
+
end
|
791
|
+
|
792
|
+
context ' with Tempfile' do
|
793
|
+
let(:io_delegate) { Tempfile.new('masamune') }
|
794
|
+
it { is_expected.to eq(io_delegate.path) }
|
795
|
+
end
|
796
|
+
end
|
599
797
|
end
|
600
798
|
|
601
799
|
describe Masamune::Schema::Map::JSONEncoder do
|
data/spec/masamune/thor_spec.rb
CHANGED
@@ -101,6 +101,7 @@ describe Masamune::Thor do
|
|
101
101
|
it 'exits with status code 0 and prints version' do
|
102
102
|
expect { cli_invocation }.to raise_error { |e|
|
103
103
|
expect(e).to be_a(SystemExit)
|
104
|
+
expect(e.message).to eq('exit')
|
104
105
|
expect(e.status).to eq(0)
|
105
106
|
}
|
106
107
|
expect(stdout.string).to match(/\Amasamune/)
|
@@ -206,13 +207,13 @@ describe Masamune::Thor do
|
|
206
207
|
it 'exits with status code 1 and prints error to stderr' do
|
207
208
|
expect { cli_invocation }.to raise_error { |e|
|
208
209
|
expect(e).to be_a(SystemExit)
|
210
|
+
expect(e.message).to eq('Path :unknown_dir not defined')
|
209
211
|
expect(e.status).to eq(1)
|
210
212
|
}
|
211
213
|
expect(stdout.string).to be_blank
|
212
214
|
expect(stderr.string).to match(/Path :unknown_dir not defined/)
|
213
215
|
end
|
214
216
|
end
|
215
|
-
|
216
217
|
end
|
217
218
|
|
218
219
|
context 'with command that prints :current_dir' do
|
metadata
CHANGED
@@ -1,195 +1,195 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: masamune
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.17.
|
4
|
+
version: 0.17.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Michael Andrews
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-02-
|
11
|
+
date: 2016-02-25 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: thor
|
15
15
|
requirement: !ruby/object:Gem::Requirement
|
16
16
|
requirements:
|
17
|
-
- -
|
17
|
+
- - '>='
|
18
18
|
- !ruby/object:Gem::Version
|
19
19
|
version: '0'
|
20
20
|
type: :runtime
|
21
21
|
prerelease: false
|
22
22
|
version_requirements: !ruby/object:Gem::Requirement
|
23
23
|
requirements:
|
24
|
-
- -
|
24
|
+
- - '>='
|
25
25
|
- !ruby/object:Gem::Version
|
26
26
|
version: '0'
|
27
27
|
- !ruby/object:Gem::Dependency
|
28
28
|
name: activesupport
|
29
29
|
requirement: !ruby/object:Gem::Requirement
|
30
30
|
requirements:
|
31
|
-
- -
|
31
|
+
- - '>='
|
32
32
|
- !ruby/object:Gem::Version
|
33
33
|
version: '0'
|
34
34
|
type: :runtime
|
35
35
|
prerelease: false
|
36
36
|
version_requirements: !ruby/object:Gem::Requirement
|
37
37
|
requirements:
|
38
|
-
- -
|
38
|
+
- - '>='
|
39
39
|
- !ruby/object:Gem::Version
|
40
40
|
version: '0'
|
41
41
|
- !ruby/object:Gem::Dependency
|
42
42
|
name: tzinfo
|
43
43
|
requirement: !ruby/object:Gem::Requirement
|
44
44
|
requirements:
|
45
|
-
- -
|
45
|
+
- - '>='
|
46
46
|
- !ruby/object:Gem::Version
|
47
47
|
version: '0'
|
48
48
|
type: :runtime
|
49
49
|
prerelease: false
|
50
50
|
version_requirements: !ruby/object:Gem::Requirement
|
51
51
|
requirements:
|
52
|
-
- -
|
52
|
+
- - '>='
|
53
53
|
- !ruby/object:Gem::Version
|
54
54
|
version: '0'
|
55
55
|
- !ruby/object:Gem::Dependency
|
56
56
|
name: chronic
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
58
58
|
requirements:
|
59
|
-
- -
|
59
|
+
- - '>='
|
60
60
|
- !ruby/object:Gem::Version
|
61
61
|
version: '0'
|
62
62
|
type: :runtime
|
63
63
|
prerelease: false
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
65
65
|
requirements:
|
66
|
-
- -
|
66
|
+
- - '>='
|
67
67
|
- !ruby/object:Gem::Version
|
68
68
|
version: '0'
|
69
69
|
- !ruby/object:Gem::Dependency
|
70
70
|
name: tilt
|
71
71
|
requirement: !ruby/object:Gem::Requirement
|
72
72
|
requirements:
|
73
|
-
- -
|
73
|
+
- - '>='
|
74
74
|
- !ruby/object:Gem::Version
|
75
75
|
version: '0'
|
76
76
|
type: :runtime
|
77
77
|
prerelease: false
|
78
78
|
version_requirements: !ruby/object:Gem::Requirement
|
79
79
|
requirements:
|
80
|
-
- -
|
80
|
+
- - '>='
|
81
81
|
- !ruby/object:Gem::Version
|
82
82
|
version: '0'
|
83
83
|
- !ruby/object:Gem::Dependency
|
84
84
|
name: erubis
|
85
85
|
requirement: !ruby/object:Gem::Requirement
|
86
86
|
requirements:
|
87
|
-
- -
|
87
|
+
- - '>='
|
88
88
|
- !ruby/object:Gem::Version
|
89
89
|
version: '0'
|
90
90
|
type: :runtime
|
91
91
|
prerelease: false
|
92
92
|
version_requirements: !ruby/object:Gem::Requirement
|
93
93
|
requirements:
|
94
|
-
- -
|
94
|
+
- - '>='
|
95
95
|
- !ruby/object:Gem::Version
|
96
96
|
version: '0'
|
97
97
|
- !ruby/object:Gem::Dependency
|
98
98
|
name: parallel
|
99
99
|
requirement: !ruby/object:Gem::Requirement
|
100
100
|
requirements:
|
101
|
-
- -
|
101
|
+
- - '>='
|
102
102
|
- !ruby/object:Gem::Version
|
103
103
|
version: '0'
|
104
104
|
type: :runtime
|
105
105
|
prerelease: false
|
106
106
|
version_requirements: !ruby/object:Gem::Requirement
|
107
107
|
requirements:
|
108
|
-
- -
|
108
|
+
- - '>='
|
109
109
|
- !ruby/object:Gem::Version
|
110
110
|
version: '0'
|
111
111
|
- !ruby/object:Gem::Dependency
|
112
112
|
name: pry
|
113
113
|
requirement: !ruby/object:Gem::Requirement
|
114
114
|
requirements:
|
115
|
-
- -
|
115
|
+
- - '>='
|
116
116
|
- !ruby/object:Gem::Version
|
117
117
|
version: '0'
|
118
118
|
type: :runtime
|
119
119
|
prerelease: false
|
120
120
|
version_requirements: !ruby/object:Gem::Requirement
|
121
121
|
requirements:
|
122
|
-
- -
|
122
|
+
- - '>='
|
123
123
|
- !ruby/object:Gem::Version
|
124
124
|
version: '0'
|
125
125
|
- !ruby/object:Gem::Dependency
|
126
126
|
name: rb-readline
|
127
127
|
requirement: !ruby/object:Gem::Requirement
|
128
128
|
requirements:
|
129
|
-
- -
|
129
|
+
- - '>='
|
130
130
|
- !ruby/object:Gem::Version
|
131
131
|
version: '0'
|
132
132
|
type: :runtime
|
133
133
|
prerelease: false
|
134
134
|
version_requirements: !ruby/object:Gem::Requirement
|
135
135
|
requirements:
|
136
|
-
- -
|
136
|
+
- - '>='
|
137
137
|
- !ruby/object:Gem::Version
|
138
138
|
version: '0'
|
139
139
|
- !ruby/object:Gem::Dependency
|
140
140
|
name: rake
|
141
141
|
requirement: !ruby/object:Gem::Requirement
|
142
142
|
requirements:
|
143
|
-
- -
|
143
|
+
- - '>='
|
144
144
|
- !ruby/object:Gem::Version
|
145
145
|
version: '0.9'
|
146
146
|
type: :development
|
147
147
|
prerelease: false
|
148
148
|
version_requirements: !ruby/object:Gem::Requirement
|
149
149
|
requirements:
|
150
|
-
- -
|
150
|
+
- - '>='
|
151
151
|
- !ruby/object:Gem::Version
|
152
152
|
version: '0.9'
|
153
153
|
- !ruby/object:Gem::Dependency
|
154
154
|
name: user_agent_parser
|
155
155
|
requirement: !ruby/object:Gem::Requirement
|
156
156
|
requirements:
|
157
|
-
- -
|
157
|
+
- - '>='
|
158
158
|
- !ruby/object:Gem::Version
|
159
159
|
version: '0'
|
160
160
|
type: :development
|
161
161
|
prerelease: false
|
162
162
|
version_requirements: !ruby/object:Gem::Requirement
|
163
163
|
requirements:
|
164
|
-
- -
|
164
|
+
- - '>='
|
165
165
|
- !ruby/object:Gem::Version
|
166
166
|
version: '0'
|
167
167
|
- !ruby/object:Gem::Dependency
|
168
168
|
name: rspec
|
169
169
|
requirement: !ruby/object:Gem::Requirement
|
170
170
|
requirements:
|
171
|
-
- -
|
171
|
+
- - '>'
|
172
172
|
- !ruby/object:Gem::Version
|
173
173
|
version: '2.99'
|
174
174
|
type: :development
|
175
175
|
prerelease: false
|
176
176
|
version_requirements: !ruby/object:Gem::Requirement
|
177
177
|
requirements:
|
178
|
-
- -
|
178
|
+
- - '>'
|
179
179
|
- !ruby/object:Gem::Version
|
180
180
|
version: '2.99'
|
181
181
|
- !ruby/object:Gem::Dependency
|
182
182
|
name: byebug
|
183
183
|
requirement: !ruby/object:Gem::Requirement
|
184
184
|
requirements:
|
185
|
-
- -
|
185
|
+
- - '>='
|
186
186
|
- !ruby/object:Gem::Version
|
187
187
|
version: '0'
|
188
188
|
type: :development
|
189
189
|
prerelease: false
|
190
190
|
version_requirements: !ruby/object:Gem::Requirement
|
191
191
|
requirements:
|
192
|
-
- -
|
192
|
+
- - '>='
|
193
193
|
- !ruby/object:Gem::Version
|
194
194
|
version: '0'
|
195
195
|
description: Hybrid Data & Work Flow
|
@@ -426,17 +426,17 @@ require_paths:
|
|
426
426
|
- lib
|
427
427
|
required_ruby_version: !ruby/object:Gem::Requirement
|
428
428
|
requirements:
|
429
|
-
- -
|
429
|
+
- - '>='
|
430
430
|
- !ruby/object:Gem::Version
|
431
431
|
version: '0'
|
432
432
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
433
433
|
requirements:
|
434
|
-
- -
|
434
|
+
- - '>='
|
435
435
|
- !ruby/object:Gem::Version
|
436
436
|
version: '0'
|
437
437
|
requirements: []
|
438
438
|
rubyforge_project:
|
439
|
-
rubygems_version: 2.4.
|
439
|
+
rubygems_version: 2.4.8
|
440
440
|
signing_key:
|
441
441
|
specification_version: 4
|
442
442
|
summary: Hybrid Data & Work Flow
|