metacrunch 4.1.0 → 4.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/metacrunch/cli.rb +51 -53
- data/lib/metacrunch/job.rb +106 -108
- data/lib/metacrunch/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c57943da0f6fc1236b58f06326223070fd1cf20d
|
4
|
+
data.tar.gz: 0a905216b6a340aed1fe95425711877cfd3384ea
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: '086cafcc6d00451da522d74a57379fa9be25538b77f4733be5ee3a1923ea38933394e00dd111fdbdfba43055989db2ba0e2879b5145ae3aa7fb8060e8f6295db'
|
7
|
+
data.tar.gz: 19c2bfaf59bd1e695aa5569f12037448ce19a52cde3c93a62b9afa99b3c4269fc5efba1042b7321d255c099ae8b44cff269d25ad9b14b44928f9983badfae61b
|
data/lib/metacrunch/cli.rb
CHANGED
@@ -1,72 +1,70 @@
|
|
1
1
|
require "optparse"
|
2
2
|
|
3
|
-
|
4
|
-
class Cli
|
3
|
+
class Metacrunch::Cli
|
5
4
|
|
6
|
-
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
5
|
+
def run
|
6
|
+
# Parse global options on order
|
7
|
+
job_argv = global_parser.order(ARGV)
|
8
|
+
# The first of the unparsed arguments is by definition the filename
|
9
|
+
# of the job.
|
10
|
+
job_file = job_argv[0]
|
11
|
+
# Manipulate ARGV so that the option handling for the job can work
|
12
|
+
ARGV.clear
|
13
|
+
job_argv[1..-1]&.each {|arg| ARGV << arg}
|
14
|
+
# Delete the old separator symbol for backward compatability
|
15
|
+
ARGV.delete_if{|arg| arg == "@@"}
|
16
|
+
# Finally run the job
|
17
|
+
run!(job_file)
|
18
|
+
rescue OptionParser::ParseError => e
|
19
|
+
error(e.message)
|
20
|
+
end
|
22
21
|
|
23
|
-
|
22
|
+
private
|
24
23
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
24
|
+
def global_parser
|
25
|
+
@global_parser ||= OptionParser.new do |opts|
|
26
|
+
opts.banner = <<-BANNER.strip_heredoc
|
27
|
+
#{ColorizedString["Usage:"].bold}
|
29
28
|
|
30
|
-
|
29
|
+
metacrunch [options] JOB_FILE [job-options] [ARGS...]
|
31
30
|
|
32
|
-
|
33
|
-
|
31
|
+
#{ColorizedString["Options:"].bold}
|
32
|
+
BANNER
|
34
33
|
|
35
|
-
|
36
|
-
|
37
|
-
end
|
34
|
+
opts.on("-v", "--version", "Show metacrunch version and exit") do
|
35
|
+
show_version
|
38
36
|
end
|
39
37
|
end
|
38
|
+
end
|
40
39
|
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
40
|
+
def show_version
|
41
|
+
puts Metacrunch::VERSION
|
42
|
+
exit(0)
|
43
|
+
end
|
45
44
|
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
45
|
+
def error(message)
|
46
|
+
puts ColorizedString["Error: #{message}\n"].red.bold
|
47
|
+
puts global_parser.help
|
48
|
+
exit(0)
|
49
|
+
end
|
51
50
|
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
58
|
-
|
59
|
-
|
51
|
+
def run!(job_file)
|
52
|
+
if job_file.blank?
|
53
|
+
error "You need to provide a job file."
|
54
|
+
elsif !File.exists?(job_file)
|
55
|
+
error "The file `#{job_file}` doesn't exist."
|
56
|
+
else
|
57
|
+
job_filename = File.expand_path(job_file)
|
58
|
+
dir = File.dirname(job_filename)
|
60
59
|
|
61
|
-
|
62
|
-
|
63
|
-
end
|
60
|
+
Dir.chdir(dir) do
|
61
|
+
run_job!(job_filename)
|
64
62
|
end
|
65
63
|
end
|
64
|
+
end
|
66
65
|
|
67
|
-
|
68
|
-
|
69
|
-
end
|
70
|
-
|
66
|
+
def run_job!(job_filename)
|
67
|
+
Metacrunch::Job.define(File.read(job_filename)).run
|
71
68
|
end
|
69
|
+
|
72
70
|
end
|
data/lib/metacrunch/job.rb
CHANGED
@@ -1,151 +1,149 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
require_relative "job/buffer"
|
1
|
+
class Metacrunch::Job # http://valve.github.io/blog/2013/10/26/constant-resolution-in-ruby/
|
2
|
+
require_relative "job/dsl"
|
3
|
+
require_relative "job/buffer"
|
5
4
|
|
6
|
-
|
5
|
+
attr_reader :dsl
|
7
6
|
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
end
|
7
|
+
class << self
|
8
|
+
def define(file_content = nil, &block)
|
9
|
+
self.new(file_content, &block)
|
12
10
|
end
|
11
|
+
end
|
13
12
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
@deprecator = ActiveSupport::Deprecation.new("5.0.0", "metacrunch")
|
13
|
+
def initialize(file_content = nil, &block)
|
14
|
+
@dsl = Dsl.new(self)
|
18
15
|
|
19
|
-
|
20
|
-
@dsl.instance_eval(file_content, "Check your metacrunch Job at Line")
|
21
|
-
elsif block_given?
|
22
|
-
@dsl.instance_eval(&block)
|
23
|
-
end
|
24
|
-
end
|
16
|
+
@deprecator = ActiveSupport::Deprecation.new("5.0.0", "metacrunch")
|
25
17
|
|
26
|
-
|
27
|
-
@
|
18
|
+
if file_content
|
19
|
+
@dsl.instance_eval(file_content, "Check your metacrunch Job at Line")
|
20
|
+
elsif block_given?
|
21
|
+
@dsl.instance_eval(&block)
|
28
22
|
end
|
23
|
+
end
|
29
24
|
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
end
|
25
|
+
def source
|
26
|
+
@source
|
27
|
+
end
|
34
28
|
|
35
|
-
|
36
|
-
|
37
|
-
|
29
|
+
def source=(source)
|
30
|
+
ensure_source!(source)
|
31
|
+
@source = source
|
32
|
+
end
|
38
33
|
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
end
|
34
|
+
def destination
|
35
|
+
@destination
|
36
|
+
end
|
43
37
|
|
44
|
-
|
45
|
-
|
46
|
-
|
38
|
+
def destination=(destination)
|
39
|
+
ensure_destination!(destination)
|
40
|
+
@destination = destination
|
41
|
+
end
|
47
42
|
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
end
|
43
|
+
def pre_process
|
44
|
+
@pre_process
|
45
|
+
end
|
52
46
|
|
53
|
-
|
54
|
-
|
55
|
-
|
47
|
+
def pre_process=(callable)
|
48
|
+
ensure_callable!(callable)
|
49
|
+
@pre_process = callable
|
50
|
+
end
|
56
51
|
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
end
|
52
|
+
def post_process
|
53
|
+
@post_process
|
54
|
+
end
|
61
55
|
|
62
|
-
|
63
|
-
|
64
|
-
|
56
|
+
def post_process=(callable)
|
57
|
+
ensure_callable!(callable)
|
58
|
+
@post_process = callable
|
59
|
+
end
|
65
60
|
|
66
|
-
|
67
|
-
|
61
|
+
def transformations
|
62
|
+
@transformations ||= []
|
63
|
+
end
|
68
64
|
|
69
|
-
|
70
|
-
|
71
|
-
buffer = buffer_size
|
72
|
-
end
|
65
|
+
def add_transformation(callable, buffer_size: nil, buffer: nil)
|
66
|
+
ensure_callable!(callable)
|
73
67
|
|
74
|
-
|
75
|
-
|
76
|
-
|
68
|
+
if buffer_size && buffer_size.is_a?(Numeric)
|
69
|
+
@deprecator.deprecation_warning(:buffer_size, :buffer)
|
70
|
+
buffer = buffer_size
|
71
|
+
end
|
77
72
|
|
78
|
-
|
73
|
+
if buffer
|
74
|
+
transformations << Metacrunch::Job::Buffer.new(buffer)
|
79
75
|
end
|
80
76
|
|
81
|
-
|
82
|
-
|
77
|
+
transformations << callable
|
78
|
+
end
|
83
79
|
|
84
|
-
|
85
|
-
|
86
|
-
source.each do |data|
|
87
|
-
data = run_transformations(data)
|
88
|
-
write_destination(data)
|
89
|
-
end
|
80
|
+
def run
|
81
|
+
run_pre_process
|
90
82
|
|
91
|
-
|
92
|
-
|
83
|
+
if source
|
84
|
+
# Run transformation for each data object available in source
|
85
|
+
source.each do |data|
|
86
|
+
data = run_transformations(data)
|
93
87
|
write_destination(data)
|
94
|
-
|
95
|
-
# Close destination
|
96
|
-
destination.close if destination
|
97
88
|
end
|
98
89
|
|
99
|
-
|
90
|
+
# Run all transformations a last time to flush existing buffers
|
91
|
+
data = run_transformations(nil, flush_buffers: true)
|
92
|
+
write_destination(data)
|
100
93
|
|
101
|
-
|
94
|
+
# Close destination
|
95
|
+
destination.close if destination
|
102
96
|
end
|
103
97
|
|
104
|
-
|
98
|
+
run_post_process
|
105
99
|
|
106
|
-
|
107
|
-
|
108
|
-
end
|
100
|
+
self
|
101
|
+
end
|
109
102
|
|
110
|
-
|
111
|
-
raise ArgumentError, "#{object} doesn't respond to #write." unless object.respond_to?(:write)
|
112
|
-
raise ArgumentError, "#{object} doesn't respond to #close." unless object.respond_to?(:close)
|
113
|
-
end
|
103
|
+
private
|
114
104
|
|
115
|
-
|
116
|
-
|
117
|
-
|
105
|
+
def ensure_source!(object)
|
106
|
+
raise ArgumentError, "#{object} doesn't respond to #each." unless object.respond_to?(:each)
|
107
|
+
end
|
118
108
|
|
119
|
-
|
120
|
-
|
121
|
-
|
109
|
+
def ensure_destination!(object)
|
110
|
+
raise ArgumentError, "#{object} doesn't respond to #write." unless object.respond_to?(:write)
|
111
|
+
raise ArgumentError, "#{object} doesn't respond to #close." unless object.respond_to?(:close)
|
112
|
+
end
|
122
113
|
|
123
|
-
|
124
|
-
|
125
|
-
|
114
|
+
def ensure_callable!(object)
|
115
|
+
raise ArgumentError, "#{object} doesn't respond to #call." unless object.respond_to?(:call)
|
116
|
+
end
|
117
|
+
|
118
|
+
def run_pre_process
|
119
|
+
pre_process.call if pre_process
|
120
|
+
end
|
121
|
+
|
122
|
+
def run_post_process
|
123
|
+
post_process.call if post_process
|
124
|
+
end
|
126
125
|
|
127
|
-
|
128
|
-
|
129
|
-
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
|
134
|
-
|
135
|
-
else
|
136
|
-
data = buffer.flush
|
137
|
-
end
|
126
|
+
def run_transformations(data, flush_buffers: false)
|
127
|
+
transformations.each do |transformation|
|
128
|
+
if transformation.is_a?(Buffer)
|
129
|
+
buffer = transformation
|
130
|
+
|
131
|
+
if data
|
132
|
+
data = buffer.buffer(data)
|
133
|
+
data = buffer.flush if flush_buffers
|
138
134
|
else
|
139
|
-
data =
|
135
|
+
data = buffer.flush
|
140
136
|
end
|
137
|
+
else
|
138
|
+
data = transformation.call(data) if data
|
141
139
|
end
|
142
|
-
|
143
|
-
data
|
144
140
|
end
|
145
141
|
|
146
|
-
|
147
|
-
|
148
|
-
end
|
142
|
+
data
|
143
|
+
end
|
149
144
|
|
145
|
+
def write_destination(data)
|
146
|
+
destination.write(data) if destination && data
|
150
147
|
end
|
148
|
+
|
151
149
|
end
|
data/lib/metacrunch/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: metacrunch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.1.
|
4
|
+
version: 4.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- René Sprotte
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: exe
|
12
12
|
cert_chain: []
|
13
|
-
date: 2017-10-
|
13
|
+
date: 2017-10-10 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: activesupport
|