scout-gear 10.11.4 → 10.11.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. checksums.yaml +4 -4
  2. data/.vimproject +17 -2
  3. data/VERSION +1 -1
  4. data/bin/scout +10 -10
  5. data/lib/scout/association/fields.rb +15 -15
  6. data/lib/scout/association/index.rb +6 -6
  7. data/lib/scout/association/item.rb +18 -8
  8. data/lib/scout/association.rb +4 -4
  9. data/lib/scout/entity/identifiers.rb +5 -5
  10. data/lib/scout/entity/property.rb +2 -2
  11. data/lib/scout/entity.rb +1 -1
  12. data/lib/scout/knowledge_base/description.rb +10 -10
  13. data/lib/scout/knowledge_base/entity.rb +6 -6
  14. data/lib/scout/knowledge_base/list.rb +1 -1
  15. data/lib/scout/knowledge_base/query.rb +4 -4
  16. data/lib/scout/knowledge_base/registry.rb +6 -6
  17. data/lib/scout/knowledge_base/traverse.rb +7 -40
  18. data/lib/scout/persist/engine/fix_width_table.rb +6 -6
  19. data/lib/scout/persist/engine/packed_index.rb +2 -2
  20. data/lib/scout/persist/engine/sharder.rb +4 -4
  21. data/lib/scout/persist/engine/tkrzw.rb +1 -1
  22. data/lib/scout/persist/engine/tokyocabinet.rb +2 -2
  23. data/lib/scout/persist/tsv/adapter/fix_width_table.rb +1 -1
  24. data/lib/scout/persist/tsv/adapter/packed_index.rb +1 -1
  25. data/lib/scout/persist/tsv/adapter/tkrzw.rb +1 -1
  26. data/lib/scout/persist/tsv/adapter/tokyocabinet.rb +3 -3
  27. data/lib/scout/persist/tsv/serialize.rb +3 -3
  28. data/lib/scout/persist/tsv.rb +1 -1
  29. data/lib/scout/semaphore.rb +100 -17
  30. data/lib/scout/tsv/annotation/repo.rb +4 -4
  31. data/lib/scout/tsv/annotation.rb +2 -2
  32. data/lib/scout/tsv/attach.rb +7 -7
  33. data/lib/scout/tsv/change_id/translate.rb +1 -1
  34. data/lib/scout/tsv/csv.rb +3 -3
  35. data/lib/scout/tsv/dumper.rb +8 -8
  36. data/lib/scout/tsv/index.rb +1 -1
  37. data/lib/scout/tsv/open.rb +3 -3
  38. data/lib/scout/tsv/stream.rb +2 -2
  39. data/lib/scout/tsv/traverse.rb +4 -4
  40. data/lib/scout/tsv/util/filter.rb +9 -9
  41. data/lib/scout/tsv/util/process.rb +2 -2
  42. data/lib/scout/tsv/util/reorder.rb +2 -2
  43. data/lib/scout/tsv/util/select.rb +3 -3
  44. data/lib/scout/tsv/util/unzip.rb +2 -2
  45. data/lib/scout/tsv/util.rb +1 -1
  46. data/lib/scout/tsv.rb +2 -2
  47. data/lib/scout/work_queue/socket.rb +3 -2
  48. data/lib/scout/work_queue/worker.rb +4 -4
  49. data/lib/scout/work_queue.rb +7 -7
  50. data/lib/scout/workflow/definition.rb +18 -16
  51. data/lib/scout/workflow/deployment/local.rb +81 -62
  52. data/lib/scout/workflow/deployment/orchestrator/batches.rb +66 -5
  53. data/lib/scout/workflow/deployment/orchestrator/chains.rb +47 -30
  54. data/lib/scout/workflow/deployment/orchestrator/rules.rb +3 -3
  55. data/lib/scout/workflow/deployment/orchestrator/workload.rb +11 -22
  56. data/lib/scout/workflow/deployment/scheduler/job.rb +34 -36
  57. data/lib/scout/workflow/deployment/scheduler/lfs.rb +1 -1
  58. data/lib/scout/workflow/deployment/scheduler/pbs.rb +4 -4
  59. data/lib/scout/workflow/deployment/scheduler/slurm.rb +2 -2
  60. data/lib/scout/workflow/deployment/scheduler.rb +23 -12
  61. data/lib/scout/workflow/deployment/trace.rb +2 -2
  62. data/lib/scout/workflow/documentation.rb +4 -4
  63. data/lib/scout/workflow/export.rb +1 -1
  64. data/lib/scout/workflow/path.rb +2 -2
  65. data/lib/scout/workflow/step/children.rb +1 -1
  66. data/lib/scout/workflow/step/dependencies.rb +36 -3
  67. data/lib/scout/workflow/step/info.rb +5 -19
  68. data/lib/scout/workflow/step/inputs.rb +1 -1
  69. data/lib/scout/workflow/step/progress.rb +2 -2
  70. data/lib/scout/workflow/step/provenance.rb +4 -4
  71. data/lib/scout/workflow/step/status.rb +23 -9
  72. data/lib/scout/workflow/step.rb +21 -19
  73. data/lib/scout/workflow/task/dependencies.rb +10 -3
  74. data/lib/scout/workflow/task/info.rb +3 -3
  75. data/lib/scout/workflow/task/inputs.rb +8 -8
  76. data/lib/scout/workflow/task.rb +37 -22
  77. data/lib/scout/workflow/usage.rb +13 -13
  78. data/lib/scout/workflow/util.rb +1 -1
  79. data/lib/scout/workflow.rb +6 -6
  80. data/scout-gear.gemspec +4 -3
  81. data/scout_commands/alias +1 -1
  82. data/scout_commands/batch/clean +12 -12
  83. data/scout_commands/batch/list +26 -25
  84. data/scout_commands/batch/tail +9 -5
  85. data/scout_commands/cat +1 -1
  86. data/scout_commands/doc +2 -2
  87. data/scout_commands/entity +4 -4
  88. data/scout_commands/find +1 -1
  89. data/scout_commands/kb/config +1 -1
  90. data/scout_commands/kb/entities +1 -1
  91. data/scout_commands/kb/list +1 -1
  92. data/scout_commands/kb/query +2 -2
  93. data/scout_commands/kb/register +1 -1
  94. data/scout_commands/kb/show +1 -1
  95. data/scout_commands/kb/traverse +1 -1
  96. data/scout_commands/log +6 -6
  97. data/scout_commands/resource/produce +2 -2
  98. data/scout_commands/resource/sync +1 -1
  99. data/scout_commands/system/clean +7 -7
  100. data/scout_commands/system/status +4 -4
  101. data/scout_commands/template +1 -1
  102. data/scout_commands/update +1 -1
  103. data/scout_commands/workflow/cmd +2 -1
  104. data/scout_commands/workflow/example +123 -0
  105. data/scout_commands/workflow/info +10 -1
  106. data/scout_commands/workflow/install +1 -1
  107. data/scout_commands/workflow/list +2 -2
  108. data/scout_commands/workflow/process +2 -2
  109. data/scout_commands/workflow/prov +3 -3
  110. data/scout_commands/workflow/task +36 -11
  111. data/scout_commands/workflow/trace +1 -1
  112. data/scout_commands/workflow/write_info +2 -2
  113. data/share/templates/command +1 -1
  114. data/test/scout/association/test_item.rb +5 -0
  115. data/test/scout/entity/test_property.rb +3 -3
  116. data/test/scout/knowledge_base/test_description.rb +1 -1
  117. data/test/scout/knowledge_base/test_traverse.rb +2 -2
  118. data/test/scout/persist/engine/test_packed_index.rb +6 -6
  119. data/test/scout/persist/test_tsv.rb +4 -4
  120. data/test/scout/persist/tsv/adapter/test_packed_index.rb +4 -4
  121. data/test/scout/persist/tsv/adapter/test_sharder.rb +23 -23
  122. data/test/scout/persist/tsv/adapter/test_tokyocabinet.rb +1 -1
  123. data/test/scout/persist/tsv/test_serialize.rb +1 -1
  124. data/test/scout/test_association.rb +1 -1
  125. data/test/scout/test_tsv.rb +2 -2
  126. data/test/scout/test_workflow.rb +2 -2
  127. data/test/scout/tsv/test_annotation.rb +4 -4
  128. data/test/scout/tsv/test_index.rb +1 -1
  129. data/test/scout/tsv/test_open.rb +2 -2
  130. data/test/scout/tsv/test_parser.rb +2 -2
  131. data/test/scout/tsv/test_stream.rb +1 -1
  132. data/test/scout/tsv/test_transformer.rb +1 -1
  133. data/test/scout/tsv/util/test_filter.rb +1 -1
  134. data/test/scout/tsv/util/test_melt.rb +1 -1
  135. data/test/scout/tsv/util/test_reorder.rb +1 -1
  136. data/test/scout/work_queue/test_socket.rb +3 -3
  137. data/test/scout/work_queue/test_worker.rb +2 -2
  138. data/test/scout/workflow/deployment/orchestrator/test_batches.rb +13 -3
  139. data/test/scout/workflow/deployment/orchestrator/test_chains.rb +15 -13
  140. data/test/scout/workflow/deployment/orchestrator/test_workload.rb +1 -1
  141. data/test/scout/workflow/deployment/test_local.rb +2 -2
  142. data/test/scout/workflow/deployment/test_scheduler.rb +1 -2
  143. data/test/scout/workflow/step/test_children.rb +1 -1
  144. data/test/scout/workflow/step/test_dependencies.rb +36 -1
  145. data/test/scout/workflow/step/test_info.rb +3 -35
  146. data/test/scout/workflow/step/test_load.rb +1 -1
  147. data/test/scout/workflow/step/test_provenance.rb +1 -1
  148. data/test/scout/workflow/step/test_status.rb +33 -1
  149. data/test/scout/workflow/task/test_dependencies.rb +9 -7
  150. data/test/scout/workflow/task/test_inputs.rb +1 -1
  151. data/test/scout/workflow/test_definition.rb +1 -1
  152. data/test/scout/workflow/test_documentation.rb +1 -1
  153. data/test/scout/workflow/test_entity.rb +2 -2
  154. data/test/scout/workflow/test_step.rb +13 -13
  155. data/test/scout/workflow/test_usage.rb +1 -1
  156. data/test/test_helper.rb +1 -1
  157. metadata +3 -2
@@ -9,7 +9,7 @@ class TestStepProvenance < Test::Unit::TestCase
9
9
  s.length
10
10
  end
11
11
 
12
- step2 = Step.new tmpfile.step2 do
12
+ step2 = Step.new tmpfile.step2 do
13
13
  step1 = dependencies.first
14
14
  step1.inputs.first + " has " + step1.load.to_s + " characters"
15
15
  end
@@ -8,7 +8,7 @@ class TestStepStatus < Test::Unit::TestCase
8
8
  s.length
9
9
  end
10
10
 
11
- step2 = Step.new tmpfile.step2 do
11
+ step2 = Step.new tmpfile.step2 do
12
12
  step1 = dependencies.first
13
13
  step1.inputs.first + " has " + step1.load.to_s + " characters"
14
14
  end
@@ -26,5 +26,37 @@ class TestStepStatus < Test::Unit::TestCase
26
26
  end
27
27
  end
28
28
 
29
+ def test_updated_file
30
+ TmpFile.with_file 'foo' do |tmpfile|
31
+ s = Step.new tmpfile
32
+ assert s.updated?
33
+ end
34
+ end
35
+
36
+ def test_deleted_dependency
37
+ tmpfile = tmpdir.test_step
38
+ step1 = Step.new tmpfile.step1, ["12"] do |s|
39
+ s.length
40
+ end
41
+
42
+ step2 = Step.new tmpfile.step2 do
43
+ step1 = dependencies.first
44
+ step1.inputs.first + " has " + step1.load.to_s + " characters"
45
+ end
46
+
47
+ step2.dependencies = [step1]
48
+
49
+ Misc.with_env "SCOUT_UPDATE", "true" do
50
+ step2.run
51
+ assert step2.updated?
52
+
53
+ sleep 0.1
54
+ step1.clean
55
+ assert step2.updated?
56
+ step1.run
57
+ refute step2.updated?
58
+ end
59
+ end
60
+
29
61
  end
30
62
 
@@ -20,7 +20,7 @@ class TestTaskDependencies < Test::Unit::TestCase
20
20
 
21
21
  assert_equal 18, wf.job(:step2, :input1 => 2, "TaskInputs#step1" => step1_job).exec
22
22
 
23
- assert_equal [step1_job.path], wf.job(:step2, :input1 => 2, "TaskInputs#step1" => step1_job).overriden_deps.collect{|d| d.path }
23
+ assert_equal [step1_job.path], wf.job(:step2, :input1 => 2, "TaskInputs#step1" => step1_job).overrider_deps.collect{|d| d.path }
24
24
 
25
25
  assert_equal 18, wf.job(:step2, "TaskInputs#step1" => step1_job).exec
26
26
  end
@@ -68,7 +68,7 @@ class TestTaskDependencies < Test::Unit::TestCase
68
68
 
69
69
  dep :my_sum
70
70
  task :double => :integer do
71
- step(:my_sum).load * 2
71
+ step(:my_sum).load * 2
72
72
  end
73
73
  end
74
74
 
@@ -90,6 +90,8 @@ class TestTaskDependencies < Test::Unit::TestCase
90
90
  job = wf.job(:double, "TaskInputs#step1" => step1)
91
91
  assert_equal 14, job.run
92
92
  assert_not_equal Task::DEFAULT_NAME, job.name
93
+
94
+
93
95
  end
94
96
 
95
97
  def test_input_dep_override
@@ -330,7 +332,7 @@ class TestTaskDependencies < Test::Unit::TestCase
330
332
  def test_can_fail
331
333
  wf = Workflow.annonymous_workflow "TaskInputs" do
332
334
  input :input1, :integer, "", 1
333
- task :step1 => :string do |i1|
335
+ task :step1 => :string do |i1|
334
336
  if i1 < 0
335
337
  raise ScoutException
336
338
  else
@@ -343,7 +345,7 @@ class TestTaskDependencies < Test::Unit::TestCase
343
345
  if step(:step1).error?
344
346
  "0"
345
347
  else
346
- step(:step1).load
348
+ step(:step1).load
347
349
  end
348
350
  end
349
351
  end
@@ -356,7 +358,7 @@ class TestTaskDependencies < Test::Unit::TestCase
356
358
  def test_jobname_input
357
359
  wf = Workflow.annonymous_workflow "JobnameInput" do
358
360
  input :input1, :string, "", nil, jobname: true
359
- task :step1 => :string do |i1|
361
+ task :step1 => :string do |i1|
360
362
  i1
361
363
  end
362
364
 
@@ -381,7 +383,7 @@ class TestTaskDependencies < Test::Unit::TestCase
381
383
  def test_jobname_input_reset
382
384
  wf = Workflow.annonymous_workflow "JobnameInput" do
383
385
  input :input1, :string, "", nil, jobname: true
384
- task :step1 => :string do |i1|
386
+ task :step1 => :string do |i1|
385
387
  i1
386
388
  end
387
389
 
@@ -404,7 +406,7 @@ class TestTaskDependencies < Test::Unit::TestCase
404
406
  def test_non_default_inputs_ignore_hidden
405
407
  wf = Workflow.annonymous_workflow "JobnameInput" do
406
408
  input :input1, :string, "", nil, jobname: true
407
- task :step1 => :string do |i1|
409
+ task :step1 => :string do |i1|
408
410
  i1
409
411
  end
410
412
 
@@ -38,7 +38,7 @@ class TestTaskInput < Test::Unit::TestCase
38
38
  task = self.example_task
39
39
 
40
40
  assert_equal [:integer], task.assign_inputs(:string => "String", :integer => 2, :integer_array => %w(1 2))[1]
41
-
41
+
42
42
  TmpFile.with_file("1\n2") do |integer_array_file|
43
43
  assert_equal [:integer], task.assign_inputs(:string => "String", :integer => 2, :integer_array => integer_array_file)[1]
44
44
  end
@@ -56,7 +56,7 @@ class TestWorkflowDefinition < Test::Unit::TestCase
56
56
  end
57
57
 
58
58
  old_cache = Scout::Config::CACHE.dup
59
- Scout::Config.set({:forget_dep_tasks => true, :remove_dep_tasks => true}, 'task:CallName#call_miguel')
59
+ Scout::Config.set({:forget_dep_tasks => true, :remove_dep_tasks => true}, 'task:CallName#call_miguel')
60
60
  job = wf.job(:call_miguel)
61
61
  dep_path = job.step(:call_name).path
62
62
  assert_equal "Hi Miguel", job.run
@@ -4,7 +4,7 @@ require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1
4
4
  class TestWorkflowDocumentation < Test::Unit::TestCase
5
5
  module UsageWorkflow
6
6
  extend Workflow
7
-
7
+
8
8
  self.name = "UsageWorkflow"
9
9
 
10
10
  self.title = "Workflow to test documentation"
@@ -39,7 +39,7 @@ class TestWorkflowEntity < Test::Unit::TestCase
39
39
 
40
40
  def test_list
41
41
  ewf = get_EWF
42
-
42
+
43
43
  l = ewf.setup(["Miki", "Clei"])
44
44
 
45
45
  assert_equal 2, l.hi.length
@@ -49,7 +49,7 @@ class TestWorkflowEntity < Test::Unit::TestCase
49
49
 
50
50
  def test_multiple
51
51
  ewf = get_EWF
52
-
52
+
53
53
  l = ewf.setup(["Miki", "Clei"])
54
54
 
55
55
  assert_equal 2, l.bye.length
@@ -20,7 +20,7 @@ class TestWorkflowStep < Test::Unit::TestCase
20
20
  s.length
21
21
  end
22
22
 
23
- step2 = Step.new tmpfile.step2 do
23
+ step2 = Step.new tmpfile.step2 do
24
24
  step1 = dependencies.first
25
25
  step1.inputs.first + " has " + step1.load.to_s + " characters"
26
26
  end
@@ -37,7 +37,7 @@ class TestWorkflowStep < Test::Unit::TestCase
37
37
  s.length
38
38
  end
39
39
 
40
- step2 = Step.new tmpdir.test_task2.step2 do
40
+ step2 = Step.new tmpdir.test_task2.step2 do
41
41
  step1 = dependencies.first
42
42
  step1.inputs.first + " has " + step1.load.to_s + " characters"
43
43
  end
@@ -86,7 +86,7 @@ class TestWorkflowStep < Test::Unit::TestCase
86
86
  assert res.nil?
87
87
  step1.clean
88
88
 
89
- step2 = Step.new tmpfile.step2 do
89
+ step2 = Step.new tmpfile.step2 do
90
90
  step1 = dependencies.first
91
91
  stream = step1.stream
92
92
 
@@ -145,7 +145,7 @@ class TestWorkflowStep < Test::Unit::TestCase
145
145
  end
146
146
  step1.type = :array
147
147
 
148
- step2 = Step.new tmpfile.step2 do
148
+ step2 = Step.new tmpfile.step2 do
149
149
  step1 = dependencies.first
150
150
  stream = step1.stream
151
151
 
@@ -160,7 +160,7 @@ class TestWorkflowStep < Test::Unit::TestCase
160
160
  step2.type = :array
161
161
  step2.dependencies = [step1]
162
162
 
163
- step3 = Step.new tmpfile.step3 do
163
+ step3 = Step.new tmpfile.step3 do
164
164
  step1, step2 = dependencies
165
165
  stream = step2.stream
166
166
 
@@ -201,7 +201,7 @@ class TestWorkflowStep < Test::Unit::TestCase
201
201
  end
202
202
  step1.type = :array
203
203
 
204
- step2 = Step.new tmpfile.step2 do
204
+ step2 = Step.new tmpfile.step2 do
205
205
  step1 = dependencies.first
206
206
  stream = step1.stream
207
207
 
@@ -216,7 +216,7 @@ class TestWorkflowStep < Test::Unit::TestCase
216
216
  step2.type = :array
217
217
  step2.dependencies = [step1]
218
218
 
219
- step3 = Step.new tmpfile.step3 do
219
+ step3 = Step.new tmpfile.step3 do
220
220
  step1 = dependencies.first
221
221
  stream = step1.stream
222
222
 
@@ -232,7 +232,7 @@ class TestWorkflowStep < Test::Unit::TestCase
232
232
  step3.dependencies = [step1]
233
233
 
234
234
 
235
- step4 = Step.new tmpfile.step4 do
235
+ step4 = Step.new tmpfile.step4 do
236
236
  step2, step3 = dependencies
237
237
 
238
238
  mutex = Mutex.new
@@ -288,7 +288,7 @@ class TestWorkflowStep < Test::Unit::TestCase
288
288
  step1.type = :array
289
289
 
290
290
 
291
- step2 = Step.new tmpfile.step2 do
291
+ step2 = Step.new tmpfile.step2 do
292
292
  step1 = dependencies.first
293
293
  stream = step1.stream
294
294
 
@@ -303,7 +303,7 @@ class TestWorkflowStep < Test::Unit::TestCase
303
303
  step2.type = :array
304
304
  step2.dependencies = [step1]
305
305
 
306
- step3 = Step.new tmpfile.step3 do
306
+ step3 = Step.new tmpfile.step3 do
307
307
  step1 = dependencies.first
308
308
  stream = step1.stream
309
309
 
@@ -319,7 +319,7 @@ class TestWorkflowStep < Test::Unit::TestCase
319
319
  step3.dependencies = [step1]
320
320
 
321
321
 
322
- step4 = Step.new tmpfile.step4 do
322
+ step4 = Step.new tmpfile.step4 do
323
323
  step2, step3 = dependencies
324
324
 
325
325
  mutex = Mutex.new
@@ -356,7 +356,7 @@ class TestWorkflowStep < Test::Unit::TestCase
356
356
  s.length
357
357
  end
358
358
 
359
- step2 = Step.new tmpfile.step2 do
359
+ step2 = Step.new tmpfile.step2 do
360
360
  step1 = dependencies.first
361
361
  step1.inputs.first + " has " + step1.load.to_s + " characters"
362
362
  end
@@ -375,7 +375,7 @@ class TestWorkflowStep < Test::Unit::TestCase
375
375
  "done1"
376
376
  end
377
377
 
378
- step2 = Step.new tmpfile.step2 do
378
+ step2 = Step.new tmpfile.step2 do
379
379
  sleep 2
380
380
  "done2"
381
381
  end
@@ -5,7 +5,7 @@ require 'scout/workflow'
5
5
  class TestWorkflowUsage < Test::Unit::TestCase
6
6
  module UsageWorkflow
7
7
  extend Workflow
8
-
8
+
9
9
  self.name = "UsageWorkflow"
10
10
 
11
11
  self.title = "Workflow to test documentation"
data/test/test_helper.rb CHANGED
@@ -54,7 +54,7 @@ class Test::Unit::TestCase
54
54
  Workflow.job_cache.clear
55
55
  SchedulerJob.batch_base_dir = tmpdir.batch
56
56
  end
57
-
57
+
58
58
  teardown do
59
59
  Open.rm_rf tmpdir
60
60
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: scout-gear
3
3
  version: !ruby/object:Gem::Version
4
- version: 10.11.4
4
+ version: 10.11.7
5
5
  platform: ruby
6
6
  authors:
7
7
  - Miguel Vazquez
@@ -252,6 +252,7 @@ files:
252
252
  - scout_commands/template
253
253
  - scout_commands/update
254
254
  - scout_commands/workflow/cmd
255
+ - scout_commands/workflow/example
255
256
  - scout_commands/workflow/info
256
257
  - scout_commands/workflow/install
257
258
  - scout_commands/workflow/list
@@ -381,7 +382,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
381
382
  - !ruby/object:Gem::Version
382
383
  version: '0'
383
384
  requirements: []
384
- rubygems_version: 3.7.0.dev
385
+ rubygems_version: 3.7.2
385
386
  specification_version: 4
386
387
  summary: basic gear for scouts
387
388
  test_files: []