metacrunch 4.1.0 → 4.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/metacrunch/cli.rb +51 -53
- data/lib/metacrunch/job.rb +106 -108
- data/lib/metacrunch/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c57943da0f6fc1236b58f06326223070fd1cf20d
|
4
|
+
data.tar.gz: 0a905216b6a340aed1fe95425711877cfd3384ea
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: '086cafcc6d00451da522d74a57379fa9be25538b77f4733be5ee3a1923ea38933394e00dd111fdbdfba43055989db2ba0e2879b5145ae3aa7fb8060e8f6295db'
|
7
|
+
data.tar.gz: 19c2bfaf59bd1e695aa5569f12037448ce19a52cde3c93a62b9afa99b3c4269fc5efba1042b7321d255c099ae8b44cff269d25ad9b14b44928f9983badfae61b
|
data/lib/metacrunch/cli.rb
CHANGED
@@ -1,72 +1,70 @@
|
|
1
1
|
require "optparse"
|
2
2
|
|
3
|
-
|
4
|
-
class Cli
|
3
|
+
class Metacrunch::Cli
|
5
4
|
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
5
|
+
def run
|
6
|
+
# Parse global options on order
|
7
|
+
job_argv = global_parser.order(ARGV)
|
8
|
+
# The first of the unparsed arguments is by definition the filename
|
9
|
+
# of the job.
|
10
|
+
job_file = job_argv[0]
|
11
|
+
# Manipulate ARGV so that the option handling for the job can work
|
12
|
+
ARGV.clear
|
13
|
+
job_argv[1..-1]&.each {|arg| ARGV << arg}
|
14
|
+
# Delete the old separator symbol for backward compatability
|
15
|
+
ARGV.delete_if{|arg| arg == "@@"}
|
16
|
+
# Finally run the job
|
17
|
+
run!(job_file)
|
18
|
+
rescue OptionParser::ParseError => e
|
19
|
+
error(e.message)
|
20
|
+
end
|
22
21
|
|
23
|
-
|
22
|
+
private
|
24
23
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
24
|
+
def global_parser
|
25
|
+
@global_parser ||= OptionParser.new do |opts|
|
26
|
+
opts.banner = <<-BANNER.strip_heredoc
|
27
|
+
#{ColorizedString["Usage:"].bold}
|
29
28
|
|
30
|
-
|
29
|
+
metacrunch [options] JOB_FILE [job-options] [ARGS...]
|
31
30
|
|
32
|
-
|
33
|
-
|
31
|
+
#{ColorizedString["Options:"].bold}
|
32
|
+
BANNER
|
34
33
|
|
35
|
-
|
36
|
-
|
37
|
-
end
|
34
|
+
opts.on("-v", "--version", "Show metacrunch version and exit") do
|
35
|
+
show_version
|
38
36
|
end
|
39
37
|
end
|
38
|
+
end
|
40
39
|
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
40
|
+
def show_version
|
41
|
+
puts Metacrunch::VERSION
|
42
|
+
exit(0)
|
43
|
+
end
|
45
44
|
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
45
|
+
def error(message)
|
46
|
+
puts ColorizedString["Error: #{message}\n"].red.bold
|
47
|
+
puts global_parser.help
|
48
|
+
exit(0)
|
49
|
+
end
|
51
50
|
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
51
|
+
def run!(job_file)
|
52
|
+
if job_file.blank?
|
53
|
+
error "You need to provide a job file."
|
54
|
+
elsif !File.exists?(job_file)
|
55
|
+
error "The file `#{job_file}` doesn't exist."
|
56
|
+
else
|
57
|
+
job_filename = File.expand_path(job_file)
|
58
|
+
dir = File.dirname(job_filename)
|
60
59
|
|
61
|
-
|
62
|
-
|
63
|
-
end
|
60
|
+
Dir.chdir(dir) do
|
61
|
+
run_job!(job_filename)
|
64
62
|
end
|
65
63
|
end
|
64
|
+
end
|
66
65
|
|
67
|
-
|
68
|
-
|
69
|
-
end
|
70
|
-
|
66
|
+
def run_job!(job_filename)
|
67
|
+
Metacrunch::Job.define(File.read(job_filename)).run
|
71
68
|
end
|
69
|
+
|
72
70
|
end
|
data/lib/metacrunch/job.rb
CHANGED
@@ -1,151 +1,149 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
require_relative "job/buffer"
|
1
|
+
class Metacrunch::Job # http://valve.github.io/blog/2013/10/26/constant-resolution-in-ruby/
|
2
|
+
require_relative "job/dsl"
|
3
|
+
require_relative "job/buffer"
|
5
4
|
|
6
|
-
|
5
|
+
attr_reader :dsl
|
7
6
|
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
end
|
7
|
+
class << self
|
8
|
+
def define(file_content = nil, &block)
|
9
|
+
self.new(file_content, &block)
|
12
10
|
end
|
11
|
+
end
|
13
12
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
@deprecator = ActiveSupport::Deprecation.new("5.0.0", "metacrunch")
|
13
|
+
def initialize(file_content = nil, &block)
|
14
|
+
@dsl = Dsl.new(self)
|
18
15
|
|
19
|
-
|
20
|
-
@dsl.instance_eval(file_content, "Check your metacrunch Job at Line")
|
21
|
-
elsif block_given?
|
22
|
-
@dsl.instance_eval(&block)
|
23
|
-
end
|
24
|
-
end
|
16
|
+
@deprecator = ActiveSupport::Deprecation.new("5.0.0", "metacrunch")
|
25
17
|
|
26
|
-
|
27
|
-
@
|
18
|
+
if file_content
|
19
|
+
@dsl.instance_eval(file_content, "Check your metacrunch Job at Line")
|
20
|
+
elsif block_given?
|
21
|
+
@dsl.instance_eval(&block)
|
28
22
|
end
|
23
|
+
end
|
29
24
|
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
end
|
25
|
+
def source
|
26
|
+
@source
|
27
|
+
end
|
34
28
|
|
35
|
-
|
36
|
-
|
37
|
-
|
29
|
+
def source=(source)
|
30
|
+
ensure_source!(source)
|
31
|
+
@source = source
|
32
|
+
end
|
38
33
|
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
end
|
34
|
+
def destination
|
35
|
+
@destination
|
36
|
+
end
|
43
37
|
|
44
|
-
|
45
|
-
|
46
|
-
|
38
|
+
def destination=(destination)
|
39
|
+
ensure_destination!(destination)
|
40
|
+
@destination = destination
|
41
|
+
end
|
47
42
|
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
end
|
43
|
+
def pre_process
|
44
|
+
@pre_process
|
45
|
+
end
|
52
46
|
|
53
|
-
|
54
|
-
|
55
|
-
|
47
|
+
def pre_process=(callable)
|
48
|
+
ensure_callable!(callable)
|
49
|
+
@pre_process = callable
|
50
|
+
end
|
56
51
|
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
end
|
52
|
+
def post_process
|
53
|
+
@post_process
|
54
|
+
end
|
61
55
|
|
62
|
-
|
63
|
-
|
64
|
-
|
56
|
+
def post_process=(callable)
|
57
|
+
ensure_callable!(callable)
|
58
|
+
@post_process = callable
|
59
|
+
end
|
65
60
|
|
66
|
-
|
67
|
-
|
61
|
+
def transformations
|
62
|
+
@transformations ||= []
|
63
|
+
end
|
68
64
|
|
69
|
-
|
70
|
-
|
71
|
-
buffer = buffer_size
|
72
|
-
end
|
65
|
+
def add_transformation(callable, buffer_size: nil, buffer: nil)
|
66
|
+
ensure_callable!(callable)
|
73
67
|
|
74
|
-
|
75
|
-
|
76
|
-
|
68
|
+
if buffer_size && buffer_size.is_a?(Numeric)
|
69
|
+
@deprecator.deprecation_warning(:buffer_size, :buffer)
|
70
|
+
buffer = buffer_size
|
71
|
+
end
|
77
72
|
|
78
|
-
|
73
|
+
if buffer
|
74
|
+
transformations << Metacrunch::Job::Buffer.new(buffer)
|
79
75
|
end
|
80
76
|
|
81
|
-
|
82
|
-
|
77
|
+
transformations << callable
|
78
|
+
end
|
83
79
|
|
84
|
-
|
85
|
-
|
86
|
-
source.each do |data|
|
87
|
-
data = run_transformations(data)
|
88
|
-
write_destination(data)
|
89
|
-
end
|
80
|
+
def run
|
81
|
+
run_pre_process
|
90
82
|
|
91
|
-
|
92
|
-
|
83
|
+
if source
|
84
|
+
# Run transformation for each data object available in source
|
85
|
+
source.each do |data|
|
86
|
+
data = run_transformations(data)
|
93
87
|
write_destination(data)
|
94
|
-
|
95
|
-
# Close destination
|
96
|
-
destination.close if destination
|
97
88
|
end
|
98
89
|
|
99
|
-
|
90
|
+
# Run all transformations a last time to flush existing buffers
|
91
|
+
data = run_transformations(nil, flush_buffers: true)
|
92
|
+
write_destination(data)
|
100
93
|
|
101
|
-
|
94
|
+
# Close destination
|
95
|
+
destination.close if destination
|
102
96
|
end
|
103
97
|
|
104
|
-
|
98
|
+
run_post_process
|
105
99
|
|
106
|
-
|
107
|
-
|
108
|
-
end
|
100
|
+
self
|
101
|
+
end
|
109
102
|
|
110
|
-
|
111
|
-
raise ArgumentError, "#{object} doesn't respond to #write." unless object.respond_to?(:write)
|
112
|
-
raise ArgumentError, "#{object} doesn't respond to #close." unless object.respond_to?(:close)
|
113
|
-
end
|
103
|
+
private
|
114
104
|
|
115
|
-
|
116
|
-
|
117
|
-
|
105
|
+
def ensure_source!(object)
|
106
|
+
raise ArgumentError, "#{object} doesn't respond to #each." unless object.respond_to?(:each)
|
107
|
+
end
|
118
108
|
|
119
|
-
|
120
|
-
|
121
|
-
|
109
|
+
def ensure_destination!(object)
|
110
|
+
raise ArgumentError, "#{object} doesn't respond to #write." unless object.respond_to?(:write)
|
111
|
+
raise ArgumentError, "#{object} doesn't respond to #close." unless object.respond_to?(:close)
|
112
|
+
end
|
122
113
|
|
123
|
-
|
124
|
-
|
125
|
-
|
114
|
+
def ensure_callable!(object)
|
115
|
+
raise ArgumentError, "#{object} doesn't respond to #call." unless object.respond_to?(:call)
|
116
|
+
end
|
117
|
+
|
118
|
+
def run_pre_process
|
119
|
+
pre_process.call if pre_process
|
120
|
+
end
|
121
|
+
|
122
|
+
def run_post_process
|
123
|
+
post_process.call if post_process
|
124
|
+
end
|
126
125
|
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
else
|
136
|
-
data = buffer.flush
|
137
|
-
end
|
126
|
+
def run_transformations(data, flush_buffers: false)
|
127
|
+
transformations.each do |transformation|
|
128
|
+
if transformation.is_a?(Buffer)
|
129
|
+
buffer = transformation
|
130
|
+
|
131
|
+
if data
|
132
|
+
data = buffer.buffer(data)
|
133
|
+
data = buffer.flush if flush_buffers
|
138
134
|
else
|
139
|
-
data =
|
135
|
+
data = buffer.flush
|
140
136
|
end
|
137
|
+
else
|
138
|
+
data = transformation.call(data) if data
|
141
139
|
end
|
142
|
-
|
143
|
-
data
|
144
140
|
end
|
145
141
|
|
146
|
-
|
147
|
-
|
148
|
-
end
|
142
|
+
data
|
143
|
+
end
|
149
144
|
|
145
|
+
def write_destination(data)
|
146
|
+
destination.write(data) if destination && data
|
150
147
|
end
|
148
|
+
|
151
149
|
end
|
data/lib/metacrunch/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: metacrunch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.1.
|
4
|
+
version: 4.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- René Sprotte
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: exe
|
12
12
|
cert_chain: []
|
13
|
-
date: 2017-10-
|
13
|
+
date: 2017-10-10 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: activesupport
|