tap 0.12.4 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/History +34 -0
- data/README +62 -41
- data/bin/tap +36 -40
- data/cmd/console.rb +14 -6
- data/cmd/manifest.rb +62 -58
- data/cmd/run.rb +49 -31
- data/doc/API +84 -0
- data/doc/Class Reference +83 -115
- data/doc/Examples/Command Line +36 -0
- data/doc/Examples/Workflow +40 -0
- data/lib/tap/app.rb +293 -214
- data/lib/tap/app/node.rb +43 -0
- data/lib/tap/app/queue.rb +77 -0
- data/lib/tap/app/stack.rb +16 -0
- data/lib/tap/app/state.rb +22 -0
- data/lib/tap/constants.rb +2 -2
- data/lib/tap/env.rb +400 -314
- data/lib/tap/env/constant.rb +227 -0
- data/lib/tap/env/gems.rb +63 -0
- data/lib/tap/env/manifest.rb +89 -0
- data/lib/tap/env/minimap.rb +292 -0
- data/lib/tap/{support → env}/string_ext.rb +2 -2
- data/lib/tap/exe.rb +113 -125
- data/lib/tap/join.rb +175 -0
- data/lib/tap/joins.rb +9 -0
- data/lib/tap/joins/switch.rb +44 -0
- data/lib/tap/joins/sync.rb +99 -0
- data/lib/tap/root.rb +100 -491
- data/lib/tap/root/utils.rb +220 -0
- data/lib/tap/{support → root}/versions.rb +31 -29
- data/lib/tap/schema.rb +248 -0
- data/lib/tap/schema/parser.rb +413 -0
- data/lib/tap/schema/utils.rb +82 -0
- data/lib/tap/support/intern.rb +19 -6
- data/lib/tap/support/templater.rb +8 -3
- data/lib/tap/task.rb +175 -171
- data/lib/tap/tasks/dump.rb +58 -0
- data/lib/tap/tasks/load.rb +62 -0
- metadata +30 -73
- data/cmd/destroy.rb +0 -27
- data/cmd/generate.rb +0 -27
- data/doc/Command Reference +0 -105
- data/doc/Syntax Reference +0 -234
- data/doc/Tutorial +0 -348
- data/lib/tap/dump.rb +0 -142
- data/lib/tap/file_task.rb +0 -384
- data/lib/tap/generator/arguments.rb +0 -13
- data/lib/tap/generator/base.rb +0 -176
- data/lib/tap/generator/destroy.rb +0 -60
- data/lib/tap/generator/generate.rb +0 -93
- data/lib/tap/generator/generators/command/command_generator.rb +0 -21
- data/lib/tap/generator/generators/command/templates/command.erb +0 -32
- data/lib/tap/generator/generators/config/config_generator.rb +0 -98
- data/lib/tap/generator/generators/generator/generator_generator.rb +0 -37
- data/lib/tap/generator/generators/generator/templates/task.erb +0 -27
- data/lib/tap/generator/generators/generator/templates/test.erb +0 -26
- data/lib/tap/generator/generators/root/root_generator.rb +0 -84
- data/lib/tap/generator/generators/root/templates/MIT-LICENSE +0 -22
- data/lib/tap/generator/generators/root/templates/README +0 -14
- data/lib/tap/generator/generators/root/templates/Rakefile +0 -84
- data/lib/tap/generator/generators/root/templates/Rapfile +0 -11
- data/lib/tap/generator/generators/root/templates/gemspec +0 -27
- data/lib/tap/generator/generators/root/templates/test/tap_test_helper.rb +0 -3
- data/lib/tap/generator/generators/task/task_generator.rb +0 -25
- data/lib/tap/generator/generators/task/templates/task.erb +0 -14
- data/lib/tap/generator/generators/task/templates/test.erb +0 -19
- data/lib/tap/generator/manifest.rb +0 -20
- data/lib/tap/generator/preview.rb +0 -69
- data/lib/tap/load.rb +0 -64
- data/lib/tap/spec.rb +0 -41
- data/lib/tap/support/aggregator.rb +0 -65
- data/lib/tap/support/audit.rb +0 -333
- data/lib/tap/support/constant.rb +0 -143
- data/lib/tap/support/constant_manifest.rb +0 -126
- data/lib/tap/support/dependencies.rb +0 -54
- data/lib/tap/support/dependency.rb +0 -44
- data/lib/tap/support/executable.rb +0 -198
- data/lib/tap/support/executable_queue.rb +0 -125
- data/lib/tap/support/gems.rb +0 -43
- data/lib/tap/support/join.rb +0 -144
- data/lib/tap/support/joins.rb +0 -12
- data/lib/tap/support/joins/switch.rb +0 -27
- data/lib/tap/support/joins/sync_merge.rb +0 -38
- data/lib/tap/support/manifest.rb +0 -171
- data/lib/tap/support/minimap.rb +0 -90
- data/lib/tap/support/node.rb +0 -176
- data/lib/tap/support/parser.rb +0 -450
- data/lib/tap/support/schema.rb +0 -385
- data/lib/tap/support/shell_utils.rb +0 -67
- data/lib/tap/test.rb +0 -77
- data/lib/tap/test/assertions.rb +0 -38
- data/lib/tap/test/env_vars.rb +0 -29
- data/lib/tap/test/extensions.rb +0 -73
- data/lib/tap/test/file_test.rb +0 -362
- data/lib/tap/test/file_test_class.rb +0 -15
- data/lib/tap/test/regexp_escape.rb +0 -87
- data/lib/tap/test/script_test.rb +0 -46
- data/lib/tap/test/script_tester.rb +0 -115
- data/lib/tap/test/subset_test.rb +0 -260
- data/lib/tap/test/subset_test_class.rb +0 -99
- data/lib/tap/test/tap_test.rb +0 -109
- data/lib/tap/test/utils.rb +0 -231
data/lib/tap/support/schema.rb
DELETED
@@ -1,385 +0,0 @@
|
|
1
|
-
require 'tap/support/node'
|
2
|
-
require 'tap/support/joins'
|
3
|
-
autoload(:Shellwords, 'shellwords')
|
4
|
-
|
5
|
-
module Tap
|
6
|
-
module Support
|
7
|
-
autoload(:Parser, 'tap/support/parser')
|
8
|
-
|
9
|
-
class Schema
|
10
|
-
module Utils
|
11
|
-
module_function
|
12
|
-
|
13
|
-
# Shell quotes the input string by enclosing in quotes if
|
14
|
-
# str has no quotes, or double quotes if str has no double
|
15
|
-
# quotes. Returns the str if it has not whitespace, quotes
|
16
|
-
# or double quotes.
|
17
|
-
#
|
18
|
-
# Raises an ArgumentError if str has both quotes and double
|
19
|
-
# quotes.
|
20
|
-
def shell_quote(str)
|
21
|
-
return str unless str =~ /[\s'"]/
|
22
|
-
|
23
|
-
quote = str.include?("'")
|
24
|
-
double_quote = str.include?('"')
|
25
|
-
|
26
|
-
case
|
27
|
-
when !quote then "'#{str}'"
|
28
|
-
when !double_quote then "\"#{str}\""
|
29
|
-
else raise ArgumentError, "cannot shell quote: #{str}"
|
30
|
-
end
|
31
|
-
end
|
32
|
-
|
33
|
-
# Formats a round string.
|
34
|
-
#
|
35
|
-
# format_round(1, [1,2,3]) # => "+1[1,2,3]"
|
36
|
-
#
|
37
|
-
def format_round(round, indicies)
|
38
|
-
"+#{round}[#{indicies.join(',')}]"
|
39
|
-
end
|
40
|
-
|
41
|
-
# Formats a sequence string.
|
42
|
-
#
|
43
|
-
# format_sequence(1, [2,3], {}) # => "1:2:3"
|
44
|
-
#
|
45
|
-
def format_sequence(source_index, outputs, options)
|
46
|
-
([source_index] + outputs).join(":") + format_options(options)
|
47
|
-
end
|
48
|
-
|
49
|
-
# Formats a global instance string.
|
50
|
-
#
|
51
|
-
# format_instance(1) # => "*1"
|
52
|
-
#
|
53
|
-
def format_instance(index)
|
54
|
-
"*#{index}"
|
55
|
-
end
|
56
|
-
|
57
|
-
# Formats a fork string.
|
58
|
-
#
|
59
|
-
# format_fork(1, [2,3],{}) # => "1[2,3]"
|
60
|
-
#
|
61
|
-
def format_fork(source_index, outputs, options)
|
62
|
-
"#{source_index[0]}[#{outputs.join(',')}]#{format_options(options)}"
|
63
|
-
end
|
64
|
-
|
65
|
-
# Formats a merge string (note the target index is
|
66
|
-
# provided first).
|
67
|
-
#
|
68
|
-
# format_merge(1, [2,3],{}) # => "1{2,3}"
|
69
|
-
#
|
70
|
-
def format_merge(target_index, inputs, options)
|
71
|
-
"#{target_index[0]}{#{inputs.join(',')}}#{format_options(options)}"
|
72
|
-
end
|
73
|
-
|
74
|
-
# Formats a sync_merge string (note the target index
|
75
|
-
# is provided first).
|
76
|
-
#
|
77
|
-
# format_sync_merge(1, [2,3],{}) # => "1(2,3)"
|
78
|
-
#
|
79
|
-
def format_sync_merge(target_index, inputs, options)
|
80
|
-
"#{target_index[0]}(#{inputs.join(',')})#{format_options(options)}"
|
81
|
-
end
|
82
|
-
|
83
|
-
# Formats an options hash into a string. Raises an error
|
84
|
-
# for unknown options.
|
85
|
-
#
|
86
|
-
# format_options({:iterate => true}) # => "i"
|
87
|
-
#
|
88
|
-
def format_options(options)
|
89
|
-
options_str = []
|
90
|
-
options.each_pair do |key, value|
|
91
|
-
unless config = Join.configurations[key]
|
92
|
-
raise "unknown key in: #{options} (#{key})"
|
93
|
-
end
|
94
|
-
|
95
|
-
if value
|
96
|
-
options_str << config.attributes[:short]
|
97
|
-
end
|
98
|
-
end
|
99
|
-
options_str.sort.join
|
100
|
-
end
|
101
|
-
end
|
102
|
-
|
103
|
-
include Utils
|
104
|
-
|
105
|
-
class << self
|
106
|
-
def parse(argv=ARGV)
|
107
|
-
Support::Parser.new(argv).schema
|
108
|
-
end
|
109
|
-
|
110
|
-
def load(argv)
|
111
|
-
parser = Parser.new
|
112
|
-
parser.load(argv)
|
113
|
-
parser.schema
|
114
|
-
end
|
115
|
-
|
116
|
-
# Loads a schema from the specified path. Raises an error if no such
|
117
|
-
# file existts.
|
118
|
-
def load_file(path)
|
119
|
-
argv = YAML.load_file(path)
|
120
|
-
argv ? load(argv) : new
|
121
|
-
end
|
122
|
-
end
|
123
|
-
|
124
|
-
# An array of the nodes registered in self.
|
125
|
-
attr_reader :nodes
|
126
|
-
|
127
|
-
def initialize(nodes=[])
|
128
|
-
@nodes = nodes
|
129
|
-
end
|
130
|
-
|
131
|
-
# Retrieves the node at index, or instantiates a new Node if one does
|
132
|
-
# not already exists.
|
133
|
-
def [](index)
|
134
|
-
nodes[index] ||= Node.new
|
135
|
-
end
|
136
|
-
|
137
|
-
# Returns the index of the node in nodes.
|
138
|
-
def index(node)
|
139
|
-
nodes.index(node)
|
140
|
-
end
|
141
|
-
|
142
|
-
# Shortcut to collect the indicies of each node in nodes. Returns nil if
|
143
|
-
# nodes is nil.
|
144
|
-
def indicies(nodes)
|
145
|
-
nodes ? nodes.collect {|node| index(node) } : nodes
|
146
|
-
end
|
147
|
-
|
148
|
-
# Returns an array of the argvs for each nodes.
|
149
|
-
def argvs
|
150
|
-
nodes.collect do |node|
|
151
|
-
node == nil ? nil : node.argv
|
152
|
-
end
|
153
|
-
end
|
154
|
-
|
155
|
-
# Returns a collection of nodes sorted into arrays by round.
|
156
|
-
def rounds
|
157
|
-
rounds = []
|
158
|
-
nodes.each do |node|
|
159
|
-
next unless node
|
160
|
-
round = node.round
|
161
|
-
(rounds[round] ||= []) << node if round
|
162
|
-
end
|
163
|
-
rounds
|
164
|
-
end
|
165
|
-
|
166
|
-
# Returns a collection of nodes sorted into arrays by natural round.
|
167
|
-
def natural_rounds
|
168
|
-
rounds = []
|
169
|
-
nodes.each do |node|
|
170
|
-
next unless node
|
171
|
-
round = node.natural_round
|
172
|
-
(rounds[round] ||= []) << node if round
|
173
|
-
end
|
174
|
-
rounds
|
175
|
-
end
|
176
|
-
|
177
|
-
# Returns a collection of global nodes.
|
178
|
-
def globals
|
179
|
-
globals = []
|
180
|
-
nodes.each do |node|
|
181
|
-
if node && node.global?
|
182
|
-
globals << node
|
183
|
-
end
|
184
|
-
end
|
185
|
-
globals
|
186
|
-
end
|
187
|
-
|
188
|
-
# Returns an array of joins among nodes in self.
|
189
|
-
def joins
|
190
|
-
joins = []
|
191
|
-
|
192
|
-
nodes.each do |node|
|
193
|
-
next unless node
|
194
|
-
if join = node.output_join
|
195
|
-
joins << join
|
196
|
-
end
|
197
|
-
end
|
198
|
-
|
199
|
-
nodes.each do |node|
|
200
|
-
next unless node
|
201
|
-
if join = node.input_join
|
202
|
-
joins << join
|
203
|
-
end
|
204
|
-
end
|
205
|
-
|
206
|
-
joins.uniq
|
207
|
-
end
|
208
|
-
|
209
|
-
# Sets a join between the nodes at the input and output indicies.
|
210
|
-
# Returns the new join.
|
211
|
-
def set(join_class, inputs, outputs, options={})
|
212
|
-
unless inputs && !inputs.empty?
|
213
|
-
raise ArgumentError, "no input nodes specified"
|
214
|
-
end
|
215
|
-
|
216
|
-
join = [join_class.new(options), [],[]]
|
217
|
-
|
218
|
-
inputs.each {|index| self[index].output = join }
|
219
|
-
outputs.each {|index| self[index].input = join }
|
220
|
-
|
221
|
-
join
|
222
|
-
end
|
223
|
-
|
224
|
-
# Removes all nil nodes, nodes with empty argvs, and orphaned joins.
|
225
|
-
# Additionally reassigns rounds by shifting later rounds up to fill
|
226
|
-
# any nils in the rounds array.
|
227
|
-
#
|
228
|
-
# Returns self.
|
229
|
-
#
|
230
|
-
#--
|
231
|
-
# Note: the algorithm for cleaning up joins can likely be optimized.
|
232
|
-
def cleanup
|
233
|
-
# remove nil and empty nodes
|
234
|
-
nodes.delete_if do |node|
|
235
|
-
node == nil || node.argv.empty?
|
236
|
-
end
|
237
|
-
|
238
|
-
# cleanup joins
|
239
|
-
joins.each do |join, input_nodes, output_nodes|
|
240
|
-
|
241
|
-
# remove missing output nodes
|
242
|
-
output_nodes.delete_if {|node| !nodes.include?(node) }
|
243
|
-
|
244
|
-
# remove missing input nodes; the removed nodes need
|
245
|
-
# to be preserved in case an orphan join results and
|
246
|
-
# the natural round before cleanup needs to be
|
247
|
-
# determined.
|
248
|
-
remaining_nodes, removed_nodes = input_nodes.partition {|node| nodes.include?(node) }
|
249
|
-
|
250
|
-
case
|
251
|
-
when remaining_nodes.empty?
|
252
|
-
# orphan join: reassign output nodes to natural round
|
253
|
-
orphan_round = Node.natural_round(removed_nodes)
|
254
|
-
output_nodes.dup.each {|node| node.round = orphan_round }
|
255
|
-
else
|
256
|
-
input_nodes.replace(remaining_nodes)
|
257
|
-
end
|
258
|
-
end
|
259
|
-
|
260
|
-
# reassign rounds
|
261
|
-
index = 0
|
262
|
-
rounds.compact.each do |round|
|
263
|
-
round.each {|node| node.round = index }
|
264
|
-
index += 1
|
265
|
-
end
|
266
|
-
|
267
|
-
self
|
268
|
-
end
|
269
|
-
|
270
|
-
def build(app)
|
271
|
-
cleanup
|
272
|
-
|
273
|
-
# instantiate the nodes
|
274
|
-
tasks = {}
|
275
|
-
nodes.each do |node|
|
276
|
-
tasks[node] = yield(node.argv) if node
|
277
|
-
end
|
278
|
-
|
279
|
-
# instantiate and reconfigure globals
|
280
|
-
instances = []
|
281
|
-
globals.each do |node|
|
282
|
-
task, args = tasks.delete(node)
|
283
|
-
instance = task.class.instance
|
284
|
-
|
285
|
-
if instances.include?(instance)
|
286
|
-
raise "global specified multple times: #{instance}"
|
287
|
-
end
|
288
|
-
|
289
|
-
instance.reconfigure(task.config.to_hash)
|
290
|
-
instance.enq(*args)
|
291
|
-
instances << instance
|
292
|
-
end
|
293
|
-
|
294
|
-
# build the workflow
|
295
|
-
joins.each do |join, input_nodes, output_nodes|
|
296
|
-
sources = input_nodes.collect {|node| tasks[node][0] }
|
297
|
-
targets = output_nodes.collect {|node| tasks[node][0] }
|
298
|
-
|
299
|
-
join.join(sources, targets)
|
300
|
-
end
|
301
|
-
|
302
|
-
# build rounds
|
303
|
-
queues = rounds.compact.collect do |round|
|
304
|
-
round.collect {|node| tasks.delete(node) }
|
305
|
-
end
|
306
|
-
|
307
|
-
# notify any args that will be overlooked
|
308
|
-
tasks.each_pair do |node, (task, args)|
|
309
|
-
next if args.empty?
|
310
|
-
warn "warning: ignoring args for node (#{index(node)}) #{task} [#{args.join(' ')}]"
|
311
|
-
end
|
312
|
-
|
313
|
-
# enque
|
314
|
-
queues.each {|queue| app.queue.concat(queue) }
|
315
|
-
app
|
316
|
-
end
|
317
|
-
|
318
|
-
# Creates an array dump of the contents of self.
|
319
|
-
def dump
|
320
|
-
cleanup
|
321
|
-
|
322
|
-
# add argvs
|
323
|
-
array = argvs
|
324
|
-
|
325
|
-
# add global declarations
|
326
|
-
globals.each do |node|
|
327
|
-
array << format_instance(index(node))
|
328
|
-
end
|
329
|
-
|
330
|
-
# add round declarations
|
331
|
-
index = 0
|
332
|
-
rounds.each do |nodes|
|
333
|
-
|
334
|
-
# skip round 0 as it is implicit
|
335
|
-
if index > 0
|
336
|
-
indicies = nodes.collect {|node| index(node) }
|
337
|
-
array << format_round(index, indicies)
|
338
|
-
end
|
339
|
-
|
340
|
-
index += 1
|
341
|
-
end
|
342
|
-
|
343
|
-
# add join declarations
|
344
|
-
joins.each do |join, input_nodes, output_nodes|
|
345
|
-
inputs = input_nodes.collect {|node| nodes.index(node) }
|
346
|
-
outputs = output_nodes.collect {|node| nodes.index(node) }
|
347
|
-
|
348
|
-
array << case join
|
349
|
-
when Joins::SyncMerge then format_sync_merge(outputs, inputs, join.options)
|
350
|
-
when Join
|
351
|
-
if inputs.length == 1
|
352
|
-
if outputs.length == 1
|
353
|
-
format_sequence(inputs, outputs, join.options)
|
354
|
-
else
|
355
|
-
format_fork(inputs, outputs, join.options)
|
356
|
-
end
|
357
|
-
else
|
358
|
-
format_merge(outputs, inputs, join.options)
|
359
|
-
end
|
360
|
-
else raise "unknown join type: #{join.class} ([#{inputs.join(',')}], [#{outputs.join(',')}])"
|
361
|
-
end
|
362
|
-
end
|
363
|
-
|
364
|
-
array
|
365
|
-
end
|
366
|
-
|
367
|
-
# Constructs a command-line string for the schema, ex:
|
368
|
-
# '-- a -- b --0:1'.
|
369
|
-
def to_s
|
370
|
-
args = []
|
371
|
-
dump.each do |obj|
|
372
|
-
if obj.kind_of?(Array)
|
373
|
-
args << "--"
|
374
|
-
args.concat obj.collect {|arg| shell_quote(arg) }
|
375
|
-
else
|
376
|
-
args << "--#{obj}"
|
377
|
-
end
|
378
|
-
end
|
379
|
-
|
380
|
-
args.join(' ')
|
381
|
-
end
|
382
|
-
|
383
|
-
end
|
384
|
-
end
|
385
|
-
end
|
@@ -1,67 +0,0 @@
|
|
1
|
-
autoload(:Tempfile, 'tempfile')
|
2
|
-
|
3
|
-
module Tap
|
4
|
-
module Support
|
5
|
-
# Provides several shell utility methods for calling programs.
|
6
|
-
#
|
7
|
-
# == Windows
|
8
|
-
# MSDOS has command line length limits specific to the version of Windows being
|
9
|
-
# run (from http://www.ss64.com/nt/cmd.html):
|
10
|
-
#
|
11
|
-
# Windows NT:: 256 characters
|
12
|
-
# Windows 2000:: 2046 characters
|
13
|
-
# Windows XP:: 8190 characters
|
14
|
-
#
|
15
|
-
# Commands longer than these limits fail, usually with something like: 'the input
|
16
|
-
# line is too long'
|
17
|
-
module ShellUtils
|
18
|
-
|
19
|
-
# Run the system command +cmd+, passing the result to the block, if given.
|
20
|
-
# Raises an error if the command fails. Uses the same semantics as
|
21
|
-
# Kernel::exec and Kernel::system.
|
22
|
-
#
|
23
|
-
# Based on FileUtils#sh from Rake.
|
24
|
-
def sh(*cmd) # :yields: ok, status
|
25
|
-
ok = system(*cmd)
|
26
|
-
|
27
|
-
if block_given?
|
28
|
-
yield(ok, $?)
|
29
|
-
else
|
30
|
-
ok or raise "Command failed with status (#{$?.exitstatus}): [#{ cmd.join(' ')}]"
|
31
|
-
end
|
32
|
-
end
|
33
|
-
|
34
|
-
# Runs the system command +cmd+ using sh, redirecting the output to the
|
35
|
-
# specified file path. Uses the redirection command:
|
36
|
-
#
|
37
|
-
# "> \"#{path}\" 2>&1 #{cmd}"
|
38
|
-
#
|
39
|
-
# This redirection has been tested on Windows, OS X, and Fedora. See
|
40
|
-
# http://www.robvanderwoude.com/redirection.html for pointers on
|
41
|
-
# redirection. The website notes that this style of redirection SHOULD
|
42
|
-
# NOT be used with commands that contain other redirections.
|
43
|
-
def redirect_sh(cmd, path, &block) # :yields: ok, status
|
44
|
-
sh( "> \"#{path}\" 2>&1 #{cmd}", &block)
|
45
|
-
end
|
46
|
-
|
47
|
-
# Runs the system command +cmd+ and returns the output as a string.
|
48
|
-
def capture_sh(cmd, quiet=false, &block) # :yields: ok, status, tempfile_path
|
49
|
-
tempfile = Tempfile.new('shell_utils')
|
50
|
-
tempfile.close
|
51
|
-
redirect_sh(cmd, tempfile.path) do |ok, status|
|
52
|
-
if block_given?
|
53
|
-
yield(ok, $?, tempfile.path)
|
54
|
-
else
|
55
|
-
ok or raise %Q{Command failed with status (#{$?.exitstatus}): [#{cmd}]
|
56
|
-
-------------- command output -------------------
|
57
|
-
#{File.read(tempfile.path)}
|
58
|
-
-------------------------------------------------
|
59
|
-
}
|
60
|
-
end
|
61
|
-
end
|
62
|
-
|
63
|
-
quiet == true ? "" : File.read(tempfile.path)
|
64
|
-
end
|
65
|
-
end
|
66
|
-
end
|
67
|
-
end
|