scout-gear 2.0.0 → 5.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (91) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +59 -2
  3. data/VERSION +1 -1
  4. data/bin/scout +231 -24
  5. data/lib/scout/cmd.rb +344 -0
  6. data/lib/scout/concurrent_stream.rb +259 -0
  7. data/lib/scout/exceptions.rb +15 -8
  8. data/lib/scout/indiferent_hash/options.rb +8 -26
  9. data/lib/scout/log/color.rb +2 -2
  10. data/lib/scout/log/fingerprint.rb +11 -1
  11. data/lib/scout/log/progress/report.rb +0 -1
  12. data/lib/scout/log/progress/util.rb +1 -1
  13. data/lib/scout/log/progress.rb +4 -4
  14. data/lib/scout/log.rb +10 -2
  15. data/lib/scout/meta_extension.rb +15 -1
  16. data/lib/scout/misc/digest.rb +56 -0
  17. data/lib/scout/misc/filesystem.rb +26 -0
  18. data/lib/scout/misc/format.rb +1 -2
  19. data/lib/scout/misc/insist.rb +56 -0
  20. data/lib/scout/misc.rb +4 -11
  21. data/lib/scout/open/lock.rb +61 -0
  22. data/lib/scout/open/remote.rb +120 -0
  23. data/lib/scout/open/stream.rb +372 -0
  24. data/lib/scout/open/util.rb +225 -0
  25. data/lib/scout/open.rb +169 -0
  26. data/lib/scout/path/find.rb +67 -21
  27. data/lib/scout/path/tmpfile.rb +8 -0
  28. data/lib/scout/path/util.rb +14 -1
  29. data/lib/scout/path.rb +6 -30
  30. data/lib/scout/persist/open.rb +17 -0
  31. data/lib/scout/persist/path.rb +15 -0
  32. data/lib/scout/persist/serialize.rb +140 -0
  33. data/lib/scout/persist.rb +54 -0
  34. data/lib/scout/resource/path.rb +15 -0
  35. data/lib/scout/resource/produce/rake.rb +69 -0
  36. data/lib/scout/resource/produce.rb +246 -0
  37. data/lib/scout/resource/scout.rb +3 -0
  38. data/lib/scout/resource.rb +37 -0
  39. data/lib/scout/simple_opt/accessor.rb +1 -1
  40. data/lib/scout/simple_opt/doc.rb +4 -22
  41. data/lib/scout/simple_opt/parse.rb +4 -3
  42. data/lib/scout/tmpfile.rb +39 -1
  43. data/lib/scout/workflow/definition.rb +72 -0
  44. data/lib/scout/workflow/documentation.rb +77 -0
  45. data/lib/scout/workflow/step/info.rb +77 -0
  46. data/lib/scout/workflow/step.rb +96 -0
  47. data/lib/scout/workflow/task/inputs.rb +112 -0
  48. data/lib/scout/workflow/task.rb +141 -0
  49. data/lib/scout/workflow/usage.rb +294 -0
  50. data/lib/scout/workflow/util.rb +11 -0
  51. data/lib/scout/workflow.rb +39 -0
  52. data/lib/scout-gear.rb +4 -0
  53. data/lib/scout.rb +1 -0
  54. data/lib/workflow-scout.rb +2 -0
  55. data/scout-gear.gemspec +66 -5
  56. data/scout_commands/alias +48 -0
  57. data/scout_commands/find +83 -0
  58. data/scout_commands/glob +0 -0
  59. data/scout_commands/rbbt +23 -0
  60. data/scout_commands/workflow/task +707 -0
  61. data/test/scout/indiferent_hash/test_options.rb +11 -1
  62. data/test/scout/misc/test_digest.rb +30 -0
  63. data/test/scout/misc/test_filesystem.rb +30 -0
  64. data/test/scout/misc/test_insist.rb +13 -0
  65. data/test/scout/open/test_lock.rb +52 -0
  66. data/test/scout/open/test_remote.rb +25 -0
  67. data/test/scout/open/test_stream.rb +515 -0
  68. data/test/scout/open/test_util.rb +73 -0
  69. data/test/scout/path/test_find.rb +28 -0
  70. data/test/scout/persist/test_open.rb +37 -0
  71. data/test/scout/persist/test_path.rb +37 -0
  72. data/test/scout/persist/test_serialize.rb +114 -0
  73. data/test/scout/resource/test_path.rb +40 -0
  74. data/test/scout/resource/test_produce.rb +62 -0
  75. data/test/scout/test_cmd.rb +85 -0
  76. data/test/scout/test_concurrent_stream.rb +29 -0
  77. data/test/scout/test_misc.rb +0 -7
  78. data/test/scout/test_open.rb +146 -0
  79. data/test/scout/test_path.rb +3 -1
  80. data/test/scout/test_persist.rb +83 -0
  81. data/test/scout/test_resource.rb +26 -0
  82. data/test/scout/test_workflow.rb +87 -0
  83. data/test/scout/workflow/step/test_info.rb +28 -0
  84. data/test/scout/workflow/task/test_inputs.rb +182 -0
  85. data/test/scout/workflow/test_step.rb +36 -0
  86. data/test/scout/workflow/test_task.rb +178 -0
  87. data/test/scout/workflow/test_usage.rb +26 -0
  88. data/test/scout/workflow/test_util.rb +17 -0
  89. data/test/test_helper.rb +17 -0
  90. data/test/test_scout-gear.rb +0 -0
  91. metadata +64 -3
@@ -0,0 +1,77 @@
1
+ class Step
2
+ def info_file
3
+ @info_file ||= @path + ".info"
4
+ end
5
+
6
+ def load_info
7
+ @info = Persist.load(info_file, :marshal) || {}
8
+ @info_load_time = Time.now
9
+ end
10
+
11
+ def save_info
12
+ Persist.save(@info, info_file, :marshal)
13
+ @info_load_time = Time.now
14
+ end
15
+
16
+ def info
17
+ outdated = @info && Open.exists?(info_file) && @info_load_time && Open.mtime(info_file) > @info_load_time
18
+
19
+ if @info.nil? || outdated
20
+ load_info
21
+ end
22
+
23
+ @info
24
+ end
25
+
26
+ def merge_info(new_info)
27
+ info = self.info
28
+ new_info.each do |key,value|
29
+ report_status new_info[:status], new_info[:message] if key == :status
30
+ if info.include?(key)
31
+ case info[key]
32
+ when Array
33
+ info[key].concat Array === value ? value : [value]
34
+ when Hash
35
+ info[key].merge! value
36
+ else
37
+ info[key] = value
38
+ end
39
+ else
40
+ info[key] = value
41
+ end
42
+ end
43
+ save_info
44
+ end
45
+
46
+ def set_info(key, value)
47
+ merge_info(key => value)
48
+ end
49
+
50
+ def init_info
51
+ @info = {
52
+ :status => :waiting
53
+ }
54
+ end
55
+
56
+ def report_status(status, message = nil)
57
+ if message.nil?
58
+ Log.info Log.color(:green, status.to_s) + " " + Log.color(:blue, path)
59
+ else
60
+ Log.info Log.color(:green, status.to_s) + " " + Log.color(:blue, path) + " " + message
61
+ end
62
+ end
63
+
64
+ def log(status, message = nil)
65
+ report_status status, message
66
+ if message
67
+ merge_info :status => status, :messages => [message]
68
+ else
69
+ merge_info :status => status
70
+ end
71
+ end
72
+
73
+ def status
74
+ info[:status]
75
+ end
76
+
77
+ end
@@ -0,0 +1,96 @@
1
+ require_relative '../path'
2
+ require_relative '../persist'
3
+ require_relative 'step/info'
4
+
5
+ class Step
6
+
7
+ attr_accessor :path, :inputs, :dependencies, :task
8
+ def initialize(path, inputs = nil, dependencies = [], &task)
9
+ @path = path
10
+ @inputs = inputs
11
+ @dependencies = dependencies
12
+ @task = task
13
+ end
14
+
15
+ attr_accessor :type
16
+ def type
17
+ @type ||= @task.respond_to?(:type) ? @task.type : nil
18
+ end
19
+
20
+ def name
21
+ @name ||= File.basename(@path)
22
+ end
23
+
24
+ def task_name
25
+ @task_name ||= @task.name if @task.respond_to?(:name)
26
+ end
27
+
28
+ def exec
29
+ self.instance_exec(*inputs, &task)
30
+ end
31
+
32
+ attr_reader :result
33
+ def run
34
+ @result = Persist.persist(name, type, :path => path) do
35
+ begin
36
+ merge_info :status => :start, :start => Time.now,
37
+ :pid => Process.pid, :pid_hostname => ENV["HOSTNAME"],
38
+ :inputs => inputs,
39
+ :dependencies => dependencies.collect{|d| d.path }
40
+
41
+ dependencies.each{|dep| dep.run }
42
+ @result = exec
43
+ ensure
44
+ if streaming?
45
+ ConcurrentStream.setup(@result) do
46
+ merge_info :status => :done, :end => Time.now
47
+ end
48
+ log :streaming
49
+ else
50
+ merge_info :status => :done, :end => Time.now
51
+ end
52
+ end
53
+ end
54
+ end
55
+
56
+ def done?
57
+ Open.exist?(path)
58
+ end
59
+
60
+ def streaming?
61
+ IO === @result || StringIO === @result
62
+ end
63
+
64
+ def join
65
+ if streaming?
66
+ Open.consume_stream(@result, false)
67
+ @result = nil
68
+ end
69
+ end
70
+
71
+ def produce
72
+ run
73
+ join
74
+ end
75
+
76
+ def load
77
+ return @result unless @result.nil? || streaming?
78
+ join
79
+ done? ? Persist.load(path, type) : exec
80
+ end
81
+
82
+ def clean
83
+ FileUtils.rm path.find if path.exist?
84
+ end
85
+
86
+ def step(task_name)
87
+ dependencies.each do |dep|
88
+ return dep if dep.task_name == task_name
89
+ end
90
+ nil
91
+ end
92
+
93
+ def digest_str
94
+ path
95
+ end
96
+ end
@@ -0,0 +1,112 @@
1
+ module Task
2
+
3
+ def format_input(value, type, options = {})
4
+ return value if IO === value || StringIO === value
5
+
6
+ value = value.load if Step === value
7
+ if String === value && ! [:path, :file].include?(type)
8
+ if Open.exists?(value)
9
+ Persist.load(value, type)
10
+ else
11
+ Persist.deserialize(value, type)
12
+ end
13
+ else
14
+ if m = type.to_s.match(/(.*)_array/)
15
+ if Array === value
16
+ value.collect{|v| format_input(v, m[1].to_sym, options) }
17
+ end
18
+ else
19
+ value
20
+ end
21
+ end
22
+ end
23
+
24
+ def assign_inputs(provided_inputs = {})
25
+ if self.inputs.nil?
26
+ case provided_inputs
27
+ when Array
28
+ return [provided_inputs, provided_inputs]
29
+ else
30
+ return [[], []]
31
+ end
32
+ end
33
+
34
+ input_array = []
35
+ non_default_inputs = []
36
+ self.inputs.each_with_index do |p,i|
37
+ name, type, desc, value, options = p
38
+ provided = Hash === provided_inputs ? provided_inputs[name] : provided_inputs[i]
39
+ provided = format_input(provided, type, options || {})
40
+ if ! provided.nil? && provided != value
41
+ non_default_inputs << name
42
+ input_array << provided
43
+ else
44
+ input_array << value
45
+ end
46
+ end
47
+
48
+ [input_array, non_default_inputs]
49
+ end
50
+
51
+ def digest_inputs(provided_inputs = {})
52
+ input_array, non_default_inputs = assign_inputs(provided_inputs)
53
+ if Array === provided_inputs
54
+ Misc.digest(input_array)
55
+ else
56
+ Misc.digest(input_array)
57
+ end
58
+ end
59
+
60
+ def process_inputs(provided_inputs = {})
61
+ input_array, non_default_inputs = assign_inputs(provided_inputs)
62
+ digest = Misc.digest(input_array)
63
+ [input_array, non_default_inputs, digest]
64
+ end
65
+
66
+ def save_file_input(orig_file, directory)
67
+ basename = File.basename(orig_file)
68
+ digest = Misc.digest(orig_file)
69
+ if basename.include? '.'
70
+ basename.sub!(/(.*)\.(.*)/, "\1-#{digest}.\2")
71
+ else
72
+ basename += "-#{digest}"
73
+ end
74
+ new_file = File.join(directory, 'saved_input_files', basename)
75
+ relative_file = File.join('.', 'saved_input_files', basename)
76
+ Open.link orig_file, new_file
77
+ relative_file
78
+ end
79
+
80
+ def save_inputs(directory, provided_inputs = {})
81
+ input_array, non_default_inputs = assign_inputs(provided_inputs)
82
+ self.inputs.each_with_index do |p,i|
83
+ name, type, desc, value, options = p
84
+ next unless non_default_inputs.include?(name)
85
+ input_file = File.join(directory, name.to_s)
86
+
87
+ if type == :file
88
+ relative_file = save_file_input(input_array[i], directory)
89
+ Persist.save(relative_file, input_file, type)
90
+ elsif type == :file_array
91
+ new_files = input_array[i].collect do |orig_file|
92
+ save_file_input(orig_file, directory)
93
+ end
94
+ Persist.save(new_files, input_file, type)
95
+ else
96
+ Persist.save(input_array[i], input_file, type)
97
+ end
98
+ end
99
+ end
100
+
101
+ def load_inputs(directory)
102
+ self.inputs.collect do |p|
103
+ name, type, desc, value, options = p
104
+ filename = File.join(directory, name.to_s)
105
+ if Open.exists?(filename) || filename = Dir.glob(File.join(filename + ".*")).first
106
+ Persist.load(filename, type)
107
+ else
108
+ value
109
+ end
110
+ end
111
+ end
112
+ end
@@ -0,0 +1,141 @@
1
+ require_relative '../meta_extension'
2
+ require_relative 'step'
3
+ require_relative 'task/inputs'
4
+
5
+ module Task
6
+ extend MetaExtension
7
+ extension_attr :name, :type, :inputs, :deps, :directory, :description
8
+
9
+ DEFAULT_NAME = "Default"
10
+
11
+ class << self
12
+ attr_accessor :default_directory
13
+
14
+ def default_directory
15
+ @default_directory ||= Path.setup('var/jobs/Task')
16
+ end
17
+ end
18
+
19
+ def directory
20
+ @directory ||= Task.default_directory
21
+ end
22
+
23
+ def exec_on(binding = self, *inputs)
24
+ binding.instance_exec(*inputs, &self)
25
+ end
26
+
27
+ def dependencies(id, provided_inputs, non_default_inputs = [])
28
+ return [] if deps.nil?
29
+ dependencies = []
30
+
31
+ provided_inputs ||= {}
32
+
33
+ load_dep = proc do |id, workflow, task, inputs, hash_options, dependencies|
34
+ task = hash_options[:task] if hash_options.include?(:task)
35
+ workflow = hash_options[:workflow] if hash_options.include?(:workflow)
36
+ id = hash_options[:id] if hash_options.include? :id
37
+
38
+ hash_inputs = hash_options.include?(:inputs)? hash_options[:inputs] : hash_options
39
+ inputs = IndiferentHash.add_defaults hash_inputs, inputs
40
+
41
+ resolved_inputs = {}
42
+ inputs.each do |k,v|
43
+ if Symbol === v
44
+ input_dep = dependencies.select{|d| d.task_name == v}.first
45
+ resolved_inputs[k] = input_dep || inputs[v] || k
46
+ else
47
+ resolved_inputs[k] = v
48
+ end
49
+ end
50
+ workflow.job(task, id, resolved_inputs)
51
+ end
52
+
53
+ deps.each do |workflow,task,options,block=nil|
54
+ if provided_inputs.include?(overriden = [workflow.name, task] * "#")
55
+ dep = provided_inputs[overriden]
56
+ dep = Step.new dep unless Step === dep
57
+ dep.type = workflow.tasks[task].type
58
+ dependencies << dep
59
+ non_default_inputs << overriden
60
+ next
61
+ end
62
+
63
+ options ||= {}
64
+ if block
65
+ inputs = IndiferentHash.add_defaults options.dup, provided_inputs
66
+
67
+ res = block.call id, inputs, dependencies
68
+
69
+ case res
70
+ when Step
71
+ dep = res
72
+ dependencies << dep
73
+ dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
74
+ non_default_inputs.concat(dep_non_default_inputs - options.keys)
75
+ when Hash
76
+ new_options = res
77
+ dep = load_dep.call(id, workflow, task, inputs, new_options, dependencies)
78
+ dependencies << dep
79
+ dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
80
+ dep_non_default_inputs -= options.keys
81
+ if new_options.include?(:inputs)
82
+ dep_non_default_inputs -= new_options[:inputs].keys
83
+ else
84
+ dep_non_default_inputs -= new_options.keys
85
+ end
86
+ non_default_inputs.concat(dep_non_default_inputs)
87
+ when Array
88
+ res.each do |_res|
89
+ if Hash === _res
90
+ new_options = _res
91
+ dep = load_dep.call(id, workflow, task, inputs, new_options, dependencies)
92
+ dependencies << dep
93
+ dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
94
+ dep_non_default_inputs -= options.keys
95
+ if new_options.include?(:inputs)
96
+ dep_non_default_inputs -= new_options[:inputs].keys
97
+ else
98
+ dep_non_default_inputs -= new_options.keys
99
+ end
100
+ non_default_inputs.concat(dep_non_default_inputs)
101
+ else
102
+ dep = _res
103
+ dependencies << dep
104
+ dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
105
+ non_default_inputs.concat(dep_non_default_inputs - options.keys)
106
+ end
107
+ end
108
+ end
109
+ else
110
+ inputs = IndiferentHash.add_defaults options.dup, provided_inputs
111
+ dep = load_dep.call(id, workflow, task, inputs, {}, dependencies)
112
+ dependencies << dep
113
+ dep_non_default_inputs = dep.task.assign_inputs(dep.inputs).last
114
+ non_default_inputs.concat(dep_non_default_inputs - options.keys)
115
+ end
116
+ end
117
+
118
+ dependencies
119
+ end
120
+
121
+ def job(id = DEFAULT_NAME, provided_inputs = nil )
122
+ provided_inputs, id = id, DEFAULT_NAME if (provided_inputs.nil? || provided_inputs.empty?) && (Hash === id || Array === id)
123
+ provided_inputs = {} if provided_inputs.nil?
124
+
125
+ inputs, non_default_inputs, input_hash = process_inputs provided_inputs
126
+
127
+ dependencies = dependencies(id, provided_inputs, non_default_inputs)
128
+
129
+ non_default_inputs.concat provided_inputs.keys.select{|k| String === k && k.include?("#") } if Hash === provided_inputs
130
+
131
+ if non_default_inputs.any?
132
+ hash = Misc.digest(:inputs => input_hash, :non_default_inputs => non_default_inputs, :dependencies => dependencies)
133
+ Log.debug "Hash #{name} - #{hash}: #{Misc.digest_str(:inputs => inputs, :dependencies => dependencies)}"
134
+ id = [id, hash] * "_"
135
+ end
136
+
137
+ path = directory[id]
138
+
139
+ Step.new path, inputs, dependencies, &self
140
+ end
141
+ end