scout-gear 10.7.1 → 10.7.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.vimproject +6 -0
- data/VERSION +1 -1
- data/lib/scout/association/index.rb +1 -1
- data/lib/scout/association.rb +21 -5
- data/lib/scout/entity/format.rb +9 -4
- data/lib/scout/entity/identifiers.rb +2 -2
- data/lib/scout/entity/named_array.rb +13 -0
- data/lib/scout/entity/property.rb +2 -1
- data/lib/scout/entity.rb +7 -4
- data/lib/scout/persist/tsv/adapter/base.rb +13 -1
- data/lib/scout/persist/tsv.rb +2 -1
- data/lib/scout/tsv/attach.rb +10 -2
- data/lib/scout/tsv/change_id.rb +3 -0
- data/lib/scout/tsv/dumper.rb +34 -30
- data/lib/scout/tsv/open.rb +1 -0
- data/lib/scout/tsv/parser.rb +21 -10
- data/lib/scout/tsv/path.rb +8 -0
- data/lib/scout/tsv/stream.rb +15 -8
- data/lib/scout/tsv/traverse.rb +12 -2
- data/lib/scout/tsv/util/process.rb +4 -1
- data/lib/scout/tsv/util/select.rb +8 -2
- data/lib/scout/tsv/util/sort.rb +23 -15
- data/lib/scout/tsv/util.rb +11 -2
- data/lib/scout/tsv.rb +23 -11
- data/lib/scout/workflow/definition.rb +3 -3
- data/lib/scout/workflow/deployment/orchestrator.rb +8 -5
- data/lib/scout/workflow/step/dependencies.rb +35 -11
- data/lib/scout/workflow/step/file.rb +2 -1
- data/lib/scout/workflow/step/info.rb +14 -2
- data/lib/scout/workflow/step/load.rb +5 -3
- data/lib/scout/workflow/step/progress.rb +6 -0
- data/lib/scout/workflow/step/provenance.rb +1 -1
- data/lib/scout/workflow/step/status.rb +10 -4
- data/lib/scout/workflow/step.rb +32 -12
- data/lib/scout/workflow/task/dependencies.rb +33 -24
- data/lib/scout/workflow/task/inputs.rb +40 -12
- data/lib/scout/workflow/task.rb +22 -10
- data/lib/scout/workflow/usage.rb +2 -2
- data/lib/scout/workflow.rb +1 -1
- data/scout-gear.gemspec +10 -4
- data/scout_commands/db/query +83 -0
- data/scout_commands/db/register +43 -0
- data/scout_commands/db/show +47 -0
- data/test/scout/entity/test_named_array.rb +21 -0
- data/test/scout/persist/test_tsv.rb +20 -0
- data/test/scout/persist/tsv/adapter/test_base.rb +20 -0
- data/test/scout/test_tsv.rb +40 -0
- data/test/scout/tsv/test_dumper.rb +24 -0
- data/test/scout/tsv/test_path.rb +24 -0
- data/test/scout/tsv/test_stream.rb +93 -0
- data/test/scout/tsv/test_traverse.rb +99 -0
- data/test/scout/tsv/test_util.rb +2 -0
- data/test/scout/tsv/util/test_select.rb +22 -0
- data/test/scout/tsv/util/test_sort.rb +24 -0
- data/test/scout/workflow/step/test_dependencies.rb +26 -0
- data/test/scout/workflow/step/test_info.rb +35 -0
- data/test/scout/workflow/task/test_dependencies.rb +67 -1
- data/test/scout/workflow/task/test_inputs.rb +24 -7
- data/test/scout/workflow/test_task.rb +36 -0
- data/test/scout/workflow/test_usage.rb +0 -1
- metadata +9 -3
@@ -9,22 +9,33 @@ module Task
|
|
9
9
|
load_dep = proc do |id, workflow, task, step_options, definition_options, dependencies|
|
10
10
|
task = step_options.delete(:task) if step_options.include?(:task)
|
11
11
|
workflow = step_options.delete(:workflow) if step_options.include?(:workflow)
|
12
|
-
|
13
|
-
|
12
|
+
|
13
|
+
step_id = id
|
14
|
+
step_id = step_options.delete(:jobname) if step_options.include?(:jobname)
|
15
|
+
step_id = nil if step_id == Task::DEFAULT_NAME
|
14
16
|
|
15
17
|
step_inputs = step_options.include?(:inputs)? step_options.delete(:inputs) : step_options
|
16
|
-
step_inputs = IndiferentHash.add_defaults step_inputs, definition_options
|
18
|
+
step_inputs = IndiferentHash.add_defaults step_inputs.dup, definition_options
|
17
19
|
|
18
20
|
resolved_inputs = {}
|
19
21
|
step_inputs.each do |k,v|
|
20
22
|
if Symbol === v
|
21
23
|
input_dep = dependencies.select{|d| d.task_name == v }.first
|
22
|
-
resolved_inputs[k] = input_dep
|
24
|
+
resolved_inputs[k] = if input_dep
|
25
|
+
input_dep
|
26
|
+
elsif provided_inputs.include?(v) && self.inputs.collect(&:first).include?(v)
|
27
|
+
provided_inputs[v]
|
28
|
+
elsif step_inputs.include?(v) && self.inputs.collect(&:first).include?(v)
|
29
|
+
step_inputs[v]
|
30
|
+
else
|
31
|
+
v
|
32
|
+
end
|
23
33
|
else
|
24
34
|
resolved_inputs[k] = v
|
25
35
|
end
|
26
36
|
end
|
27
|
-
|
37
|
+
|
38
|
+
job = workflow.job(task, step_id, resolved_inputs)
|
28
39
|
compute_options = definition_options[:compute] || []
|
29
40
|
compute_options = [compute_options] unless Array === compute_options
|
30
41
|
compute_options << :canfail if definition_options[:canfail]
|
@@ -33,26 +44,25 @@ module Task
|
|
33
44
|
compute[job.path] = compute_options if compute_options.any?
|
34
45
|
|
35
46
|
job.overriden = false if definition_options[:not_overriden]
|
47
|
+
job.compute = job.compute.nil? ? compute : job.compute.merge(compute)
|
36
48
|
|
37
49
|
[job, step_inputs]
|
38
50
|
end
|
39
51
|
|
40
52
|
# Helper function
|
41
|
-
|
53
|
+
filter_dep_non_default_inputs = proc do |dep,definition_options|
|
42
54
|
dep_non_default_inputs = dep.non_default_inputs
|
43
|
-
dep_non_default_inputs.select do |name|
|
44
|
-
step_inputs.include?(name)
|
45
|
-
end
|
46
55
|
dep_non_default_inputs.reject! do |name|
|
47
|
-
definition_options.include?(name)
|
48
|
-
(definition_options[name] != :placeholder || definition_options[name] != dep.inputs[name])
|
56
|
+
definition_options.include?(name)
|
49
57
|
end
|
50
58
|
|
51
59
|
dep_non_default_inputs
|
52
60
|
end
|
53
61
|
|
54
62
|
deps.each do |workflow,task,definition_options,block=nil|
|
55
|
-
definition_options
|
63
|
+
definition_options = definition_options.nil? ? {} : definition_options.dup
|
64
|
+
|
65
|
+
dep_id = definition_options.include?(:jobname) ? definition_options.delete(:jobname) : id
|
56
66
|
|
57
67
|
if provided_inputs.include?(overriden = [workflow.name, task] * "#")
|
58
68
|
dep = provided_inputs[overriden]
|
@@ -66,51 +76,50 @@ module Task
|
|
66
76
|
next
|
67
77
|
end
|
68
78
|
|
69
|
-
definition_options ||= {}
|
70
79
|
|
71
80
|
if block
|
72
|
-
fixed_provided_inputs = self.assign_inputs(provided_inputs).first.to_hash
|
73
|
-
self.inputs.each do |name,type,desc,value|
|
81
|
+
fixed_provided_inputs = self.assign_inputs(provided_inputs, dep_id).first.to_hash
|
82
|
+
self.inputs.each do |name,type,desc,value,options|
|
74
83
|
fixed_provided_inputs[name] = value unless fixed_provided_inputs.include?(name)
|
75
84
|
end
|
76
85
|
fixed_provided_inputs = IndiferentHash.add_defaults fixed_provided_inputs, provided_inputs
|
77
86
|
block_options = IndiferentHash.add_defaults definition_options.dup, fixed_provided_inputs
|
78
87
|
|
79
|
-
res = block.call
|
88
|
+
res = block.call dep_id, block_options, dependencies
|
80
89
|
|
81
90
|
case res
|
82
91
|
when Step
|
83
92
|
dep = res
|
84
93
|
dependencies << dep
|
85
|
-
dep_non_default_inputs =
|
94
|
+
dep_non_default_inputs = filter_dep_non_default_inputs.call(dep, definition_options)
|
86
95
|
non_default_inputs.concat(dep_non_default_inputs)
|
87
96
|
when Hash
|
88
97
|
step_options = block_options.merge(res)
|
89
|
-
dep, step_inputs = load_dep.call(
|
98
|
+
dep, step_inputs = load_dep.call(dep_id, workflow, task, step_options.dup, block_options, dependencies)
|
90
99
|
dependencies << dep
|
91
|
-
dep_non_default_inputs =
|
100
|
+
dep_non_default_inputs = filter_dep_non_default_inputs.call(dep, definition_options)
|
92
101
|
non_default_inputs.concat(dep_non_default_inputs)
|
93
102
|
when Array
|
94
103
|
res.each do |_res|
|
95
104
|
if Hash === _res
|
96
105
|
step_options = block_options.merge(_res)
|
97
|
-
dep, step_inputs = load_dep.call(
|
106
|
+
dep, step_inputs = load_dep.call(dep_id, workflow, task, step_options.dup, block_options, dependencies)
|
98
107
|
dependencies << dep
|
99
|
-
dep_non_default_inputs =
|
108
|
+
dep_non_default_inputs = filter_dep_non_default_inputs.call(dep, definition_options)
|
100
109
|
non_default_inputs.concat(dep_non_default_inputs)
|
101
110
|
else
|
102
111
|
dep = _res
|
103
112
|
dependencies << dep
|
104
|
-
dep_non_default_inputs =
|
113
|
+
dep_non_default_inputs = filter_dep_non_default_inputs.call(dep, definition_options)
|
105
114
|
non_default_inputs.concat(dep_non_default_inputs)
|
106
115
|
end
|
107
116
|
end
|
108
117
|
end
|
109
118
|
else
|
110
119
|
step_options = IndiferentHash.add_defaults definition_options.dup, provided_inputs
|
111
|
-
dep, step_inputs = load_dep.call(
|
120
|
+
dep, step_inputs = load_dep.call(dep_id, workflow, task, step_options, definition_options, dependencies)
|
112
121
|
dependencies << dep
|
113
|
-
dep_non_default_inputs =
|
122
|
+
dep_non_default_inputs = filter_dep_non_default_inputs.call(dep, definition_options)
|
114
123
|
non_default_inputs.concat(dep_non_default_inputs)
|
115
124
|
end
|
116
125
|
end
|
@@ -3,7 +3,7 @@ module Task
|
|
3
3
|
def self.format_input(value, type, options = {})
|
4
4
|
return value if IO === value || StringIO === value || Step === value
|
5
5
|
|
6
|
-
if String === value && ! [:path, :file, :folder, :binary, :tsv].include?(type) && ! (options && (options[:noload] || options[:stream] || options[:nofile]))
|
6
|
+
if String === value && ! [:path, :file, :folder, :binary, :tsv].include?(type) && ! (options && (options[:noload] || options[:stream] || options[:nofile] || options[:asfile]))
|
7
7
|
if Open.exists?(value) && ! Open.directory?(value)
|
8
8
|
Persist.load(value, type)
|
9
9
|
else
|
@@ -35,11 +35,13 @@ module Task
|
|
35
35
|
input_array = []
|
36
36
|
input_names = []
|
37
37
|
non_default_inputs = []
|
38
|
+
jobname_input = nil
|
38
39
|
self.inputs.each_with_index do |p,i|
|
39
40
|
name, type, desc, value, options = p
|
40
41
|
input_names << name
|
41
42
|
provided = Hash === provided_inputs ? provided_inputs[name] : provided_inputs[i]
|
42
43
|
provided = Task.format_input(provided, type, options || {})
|
44
|
+
|
43
45
|
if provided == value
|
44
46
|
same_as_default = true
|
45
47
|
elsif String === provided && Symbol === value && provided == value.to_s
|
@@ -49,25 +51,39 @@ module Task
|
|
49
51
|
else
|
50
52
|
same_as_default = false
|
51
53
|
end
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
elsif options && options[:jobname]
|
56
|
-
input_array << id
|
57
|
-
else
|
58
|
-
input_array << value
|
54
|
+
|
55
|
+
if options && options[:jobname] && id == provided
|
56
|
+
same_as_jobname = true
|
59
57
|
end
|
58
|
+
|
59
|
+
jobname_input = name if same_as_jobname
|
60
|
+
|
61
|
+
final = if ! provided.nil? && ! same_as_default && ! same_as_jobname
|
62
|
+
non_default_inputs << name.to_sym
|
63
|
+
provided
|
64
|
+
elsif options && options[:jobname] && id
|
65
|
+
non_default_inputs << name.to_sym
|
66
|
+
provided || id
|
67
|
+
else
|
68
|
+
value
|
69
|
+
end
|
70
|
+
|
71
|
+
final = Path.setup(final.dup) if String === final && ! (Path === final) && (type == :file || type == :path || (options && options[:asfile]))
|
72
|
+
|
73
|
+
final = final.find if (Path === final) && (type == :file)
|
74
|
+
|
75
|
+
input_array << final
|
60
76
|
end
|
61
77
|
|
62
78
|
NamedArray.setup(input_array, input_names)
|
63
79
|
|
64
|
-
[input_array, non_default_inputs]
|
80
|
+
[input_array, non_default_inputs, jobname_input]
|
65
81
|
end
|
66
82
|
|
67
83
|
def process_inputs(provided_inputs = {}, id = nil)
|
68
|
-
input_array, non_default_inputs = assign_inputs provided_inputs, id
|
84
|
+
input_array, non_default_inputs, jobname_input = assign_inputs provided_inputs, id
|
69
85
|
digest_str = Misc.digest_str(input_array)
|
70
|
-
[input_array, non_default_inputs, digest_str]
|
86
|
+
[input_array, non_default_inputs, digest_str, jobname_input]
|
71
87
|
end
|
72
88
|
|
73
89
|
def self.save_file_input(orig_file, directory)
|
@@ -139,7 +155,7 @@ module Task
|
|
139
155
|
elsif filename.end_with?('.as_path')
|
140
156
|
value = Open.read(filename).strip
|
141
157
|
Path.setup value
|
142
|
-
elsif (options && (options[:noload] || options[:stream] || options[:nofile]))
|
158
|
+
elsif (options && (options[:noload] || options[:stream] || options[:nofile] || options[:asfile]))
|
143
159
|
filename
|
144
160
|
else
|
145
161
|
Persist.load(filename, type)
|
@@ -150,12 +166,24 @@ module Task
|
|
150
166
|
end
|
151
167
|
|
152
168
|
def load_inputs(directory)
|
169
|
+
if Open.exists?(directory) && ! Open.directory?(directory)
|
170
|
+
TmpFile.with_file do |tmp_directory|
|
171
|
+
Misc.in_dir tmp_directory do
|
172
|
+
CMD.cmd("tar xvfz '#{directory}'")
|
173
|
+
end
|
174
|
+
return load_inputs(tmp_directory)
|
175
|
+
end
|
176
|
+
end
|
177
|
+
|
153
178
|
inputs = IndiferentHash.setup({})
|
179
|
+
seen = []
|
154
180
|
self.recursive_inputs.each do |p|
|
155
181
|
name, type, desc, value, options = p
|
182
|
+
next if seen.include?(name)
|
156
183
|
filename = File.join(directory, name.to_s)
|
157
184
|
value = Task.load_input_from_file(filename, type, options)
|
158
185
|
inputs[name] = value unless value.nil?
|
186
|
+
seen << name
|
159
187
|
end
|
160
188
|
|
161
189
|
Dir.glob(File.join(directory, "*#*")).each do |file|
|
data/lib/scout/workflow/task.rb
CHANGED
@@ -31,19 +31,29 @@ module Task
|
|
31
31
|
end
|
32
32
|
|
33
33
|
def job(id = nil, provided_inputs = nil)
|
34
|
-
|
34
|
+
|
35
|
+
if Hash === provided_inputs
|
36
|
+
memory_inputs = provided_inputs.values_at *self.recursive_inputs.collect{|t| t.first }.uniq
|
37
|
+
memory_inputs += provided_inputs.select{|k,v| k.to_s.include?("#") }.collect{|p| p * "=" }
|
38
|
+
memory_inputs << provided_inputs[:load_inputs]
|
39
|
+
else
|
40
|
+
memory_inputs = provided_inputs
|
41
|
+
end
|
42
|
+
|
43
|
+
Persist.memory("Task job #{self.name} #{id}", other_options: {task: self, id: id, provided_inputs: memory_inputs}) do
|
35
44
|
provided_inputs, id = id, nil if (provided_inputs.nil? || provided_inputs.empty?) && (Hash === id || Array === id)
|
36
45
|
provided_inputs = {} if provided_inputs.nil?
|
37
46
|
IndiferentHash.setup(provided_inputs)
|
38
47
|
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
end
|
44
|
-
id = DEFAULT_NAME if id.nil?
|
48
|
+
jobname_input = nil
|
49
|
+
inputs.each do |name,type,desc,default,input_options|
|
50
|
+
next unless input_options && input_options[:jobname]
|
51
|
+
jobname_input = name
|
45
52
|
end
|
46
53
|
|
54
|
+
id = provided_inputs[jobname_input] if jobname_input && id.nil?
|
55
|
+
#id = provided_inputs[:id] if provided_inputs.include?(:id)
|
56
|
+
|
47
57
|
missing_inputs = []
|
48
58
|
self.inputs.each do |input,type,desc,val,options|
|
49
59
|
next unless options && options[:required]
|
@@ -65,11 +75,13 @@ module Task
|
|
65
75
|
compute = {}
|
66
76
|
dependencies = dependencies(id, provided_inputs, non_default_inputs, compute)
|
67
77
|
|
68
|
-
#non_default_inputs.concat provided_inputs.keys.select{|k| String === k && k.include?("#") } if Hash === provided_inputs
|
69
|
-
|
70
78
|
non_default_inputs.uniq!
|
71
79
|
|
72
|
-
|
80
|
+
non_default_inputs.delete_if{|k| k.to_s.include? "#" } unless dependencies.select{|d| d.overriden? }.any?
|
81
|
+
|
82
|
+
id = DEFAULT_NAME if id.nil?
|
83
|
+
|
84
|
+
if non_default_inputs.any? && !(non_default_inputs == [jobname_input] && provided_inputs[jobname_input] == id)
|
73
85
|
hash = Misc.digest(:inputs => input_digest_str, :dependencies => dependencies)
|
74
86
|
name = [id, hash] * "_"
|
75
87
|
else
|
data/lib/scout/workflow/usage.rb
CHANGED
@@ -162,7 +162,7 @@ module Workflow
|
|
162
162
|
|
163
163
|
def _prov_tasks(tree)
|
164
164
|
tasks = []
|
165
|
-
heap =
|
165
|
+
heap = tree.values
|
166
166
|
while heap.any?
|
167
167
|
t = heap.pop
|
168
168
|
t.each do |k,v|
|
@@ -271,7 +271,7 @@ module Workflow
|
|
271
271
|
description = description.split("\n\n").first
|
272
272
|
|
273
273
|
next if abridge && ! final.include?(name)
|
274
|
-
str.puts Misc.format_definition_list_item(name.to_s, description, nil, nil, :yellow)
|
274
|
+
str.puts Misc.format_definition_list_item(name.to_s, description, nil, nil, color: :yellow)
|
275
275
|
|
276
276
|
prov_string = prov_string(dep_tree(name))
|
277
277
|
str.puts Misc.format_paragraph Log.color(:blue, "->" + prov_string) if prov_string && ! prov_string.empty?
|
data/lib/scout/workflow.rb
CHANGED
@@ -155,7 +155,7 @@ module Workflow
|
|
155
155
|
|
156
156
|
def job(name, *args)
|
157
157
|
task = tasks[name]
|
158
|
-
raise TaskNotFound, "Task #{
|
158
|
+
raise TaskNotFound, "Task #{name} in #{self.to_s}" if task.nil?
|
159
159
|
step = task.job(*args)
|
160
160
|
step.extend step_module
|
161
161
|
step
|
data/scout-gear.gemspec
CHANGED
@@ -2,16 +2,16 @@
|
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
3
3
|
# Instead, edit Juwelier::Tasks in Rakefile, and run 'rake gemspec'
|
4
4
|
# -*- encoding: utf-8 -*-
|
5
|
-
# stub: scout-gear 10.7.
|
5
|
+
# stub: scout-gear 10.7.2 ruby lib
|
6
6
|
|
7
7
|
Gem::Specification.new do |s|
|
8
8
|
s.name = "scout-gear".freeze
|
9
|
-
s.version = "10.7.
|
9
|
+
s.version = "10.7.2".freeze
|
10
10
|
|
11
11
|
s.required_rubygems_version = Gem::Requirement.new(">= 0".freeze) if s.respond_to? :required_rubygems_version=
|
12
12
|
s.require_paths = ["lib".freeze]
|
13
13
|
s.authors = ["Miguel Vazquez".freeze]
|
14
|
-
s.date = "2024-
|
14
|
+
s.date = "2024-12-10"
|
15
15
|
s.description = "Temporary files, logs, path, resources, persistence, workflows, TSV, etc.".freeze
|
16
16
|
s.email = "mikisvaz@gmail.com".freeze
|
17
17
|
s.executables = ["scout".freeze]
|
@@ -41,6 +41,7 @@ Gem::Specification.new do |s|
|
|
41
41
|
"lib/scout/entity.rb",
|
42
42
|
"lib/scout/entity/format.rb",
|
43
43
|
"lib/scout/entity/identifiers.rb",
|
44
|
+
"lib/scout/entity/named_array.rb",
|
44
45
|
"lib/scout/entity/object.rb",
|
45
46
|
"lib/scout/entity/property.rb",
|
46
47
|
"lib/scout/offsite.rb",
|
@@ -121,6 +122,9 @@ Gem::Specification.new do |s|
|
|
121
122
|
"scout_commands/alias",
|
122
123
|
"scout_commands/batch/clean",
|
123
124
|
"scout_commands/batch/list",
|
125
|
+
"scout_commands/db/query",
|
126
|
+
"scout_commands/db/register",
|
127
|
+
"scout_commands/db/show",
|
124
128
|
"scout_commands/doc",
|
125
129
|
"scout_commands/find",
|
126
130
|
"scout_commands/glob",
|
@@ -151,6 +155,7 @@ Gem::Specification.new do |s|
|
|
151
155
|
"test/scout/association/test_item.rb",
|
152
156
|
"test/scout/entity/test_format.rb",
|
153
157
|
"test/scout/entity/test_identifiers.rb",
|
158
|
+
"test/scout/entity/test_named_array.rb",
|
154
159
|
"test/scout/entity/test_object.rb",
|
155
160
|
"test/scout/entity/test_property.rb",
|
156
161
|
"test/scout/offsite/test_ssh.rb",
|
@@ -188,6 +193,7 @@ Gem::Specification.new do |s|
|
|
188
193
|
"test/scout/tsv/test_index.rb",
|
189
194
|
"test/scout/tsv/test_open.rb",
|
190
195
|
"test/scout/tsv/test_parser.rb",
|
196
|
+
"test/scout/tsv/test_path.rb",
|
191
197
|
"test/scout/tsv/test_stream.rb",
|
192
198
|
"test/scout/tsv/test_transformer.rb",
|
193
199
|
"test/scout/tsv/test_traverse.rb",
|
@@ -225,7 +231,7 @@ Gem::Specification.new do |s|
|
|
225
231
|
]
|
226
232
|
s.homepage = "http://github.com/mikisvaz/scout-gear".freeze
|
227
233
|
s.licenses = ["MIT".freeze]
|
228
|
-
s.rubygems_version = "3.5.
|
234
|
+
s.rubygems_version = "3.5.23".freeze
|
229
235
|
s.summary = "basic gear for scouts".freeze
|
230
236
|
|
231
237
|
s.specification_version = 4
|
@@ -0,0 +1,83 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'scout'
|
4
|
+
require 'scout/association'
|
5
|
+
|
6
|
+
$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
|
7
|
+
|
8
|
+
options = SOPT.setup <<EOF
|
9
|
+
|
10
|
+
Query a database
|
11
|
+
|
12
|
+
$ #{$0} [<options>] <name> <entity>
|
13
|
+
|
14
|
+
-h--help Print this help
|
15
|
+
-l--list Only list matches
|
16
|
+
-s--source* Source description
|
17
|
+
-t--target* Target description
|
18
|
+
-n--namespace* Namespace
|
19
|
+
-i--identifiers* Identifiers
|
20
|
+
EOF
|
21
|
+
if options[:help]
|
22
|
+
if defined? scout_usage
|
23
|
+
scout_usage
|
24
|
+
else
|
25
|
+
puts SOPT.doc
|
26
|
+
end
|
27
|
+
exit 0
|
28
|
+
end
|
29
|
+
|
30
|
+
name, entity = ARGV
|
31
|
+
|
32
|
+
raise MissingParameterException, :name if name.nil?
|
33
|
+
raise MissingParameterException, :entity if entity.nil?
|
34
|
+
|
35
|
+
registry = begin
|
36
|
+
Scout.var.databases.registry.yaml
|
37
|
+
rescue
|
38
|
+
{}
|
39
|
+
end
|
40
|
+
|
41
|
+
list = IndiferentHash.process_options options, :list
|
42
|
+
|
43
|
+
raise ParameterException "Database #{name} not found Options: #{Log.fingerprint registry.keys}" unless registry.include? name
|
44
|
+
|
45
|
+
file, db_options = registry[name]
|
46
|
+
options = db_options.merge(options)
|
47
|
+
|
48
|
+
IndiferentHash.setup(options)
|
49
|
+
options.keys_to_sym!
|
50
|
+
|
51
|
+
index = Association.index(file, **options)
|
52
|
+
|
53
|
+
if entity.end_with?("~")
|
54
|
+
matches = index.match(entity[0..-2])
|
55
|
+
elsif entity.start_with?("~")
|
56
|
+
index = index.reverse
|
57
|
+
matches = index.match(entity[1..-1])
|
58
|
+
reverse = true
|
59
|
+
elsif entity.include?("~")
|
60
|
+
matches = [entity]
|
61
|
+
else
|
62
|
+
matches = index.match(entity)
|
63
|
+
end
|
64
|
+
|
65
|
+
AssociationItem.setup(matches) if reverse
|
66
|
+
|
67
|
+
if matches.any?
|
68
|
+
if list
|
69
|
+
if reverse
|
70
|
+
puts matches.collect(&:invert) * "\n"
|
71
|
+
else
|
72
|
+
puts matches * "\n"
|
73
|
+
end
|
74
|
+
else
|
75
|
+
matches.each do |match|
|
76
|
+
puts Log.color :magenta, (reverse ? match.invert : match)
|
77
|
+
puts index[match].prety_print
|
78
|
+
end
|
79
|
+
end
|
80
|
+
else
|
81
|
+
STDERR.puts "Query #{entity} return no results"
|
82
|
+
end
|
83
|
+
|
@@ -0,0 +1,43 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'scout'
|
4
|
+
|
5
|
+
$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
|
6
|
+
|
7
|
+
options = SOPT.setup <<EOF
|
8
|
+
|
9
|
+
Register a database
|
10
|
+
|
11
|
+
$ #{$0} [<options>] <name> <filename>
|
12
|
+
|
13
|
+
-h--help Print this help
|
14
|
+
-s--source* Source description
|
15
|
+
-t--target* Target description
|
16
|
+
-n--namespace* Namespace
|
17
|
+
-i--identifiers* Identifiers
|
18
|
+
EOF
|
19
|
+
if options.delete :help
|
20
|
+
if defined? scout_usage
|
21
|
+
scout_usage
|
22
|
+
else
|
23
|
+
puts SOPT.doc
|
24
|
+
end
|
25
|
+
exit 0
|
26
|
+
end
|
27
|
+
|
28
|
+
name, file = ARGV
|
29
|
+
|
30
|
+
raise MissingParameterException, :name if name.nil?
|
31
|
+
raise MissingParameterException, :file if file.nil?
|
32
|
+
|
33
|
+
registry = begin
|
34
|
+
Scout.var.databases.registry.yaml
|
35
|
+
rescue
|
36
|
+
{}
|
37
|
+
end
|
38
|
+
|
39
|
+
file = Scout.identify(File.expand_path(file))
|
40
|
+
registry[name] = [Scout.identify(file), options]
|
41
|
+
|
42
|
+
Scout.var.databases.registry.write(registry.to_yaml)
|
43
|
+
|
@@ -0,0 +1,47 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require 'scout'
|
4
|
+
require 'scout/association'
|
5
|
+
|
6
|
+
$0 = "scout #{$previous_commands.any? ? $previous_commands*" " + " " : "" }#{ File.basename(__FILE__) }" if $previous_commands
|
7
|
+
|
8
|
+
options = SOPT.setup <<EOF
|
9
|
+
|
10
|
+
Show a database
|
11
|
+
|
12
|
+
$ #{$0} [<options>] <name>
|
13
|
+
|
14
|
+
-h--help Print this help
|
15
|
+
EOF
|
16
|
+
if options[:help]
|
17
|
+
if defined? scout_usage
|
18
|
+
scout_usage
|
19
|
+
else
|
20
|
+
puts SOPT.doc
|
21
|
+
end
|
22
|
+
exit 0
|
23
|
+
end
|
24
|
+
|
25
|
+
|
26
|
+
registry = begin
|
27
|
+
Scout.var.databases.registry.yaml
|
28
|
+
rescue
|
29
|
+
{}
|
30
|
+
end
|
31
|
+
|
32
|
+
name = ARGV.first
|
33
|
+
|
34
|
+
if name.nil?
|
35
|
+
puts registry.keys * "\n"
|
36
|
+
|
37
|
+
else
|
38
|
+
|
39
|
+
raise ParameterException "Database #{name} not found Options: #{Log.fingerprint registry.keys}" unless registry.include? name
|
40
|
+
|
41
|
+
file, options = registry[name]
|
42
|
+
|
43
|
+
IndiferentHash.setup(options)
|
44
|
+
options.keys_to_sym!
|
45
|
+
|
46
|
+
Log.tsv Association.database(file, **options)
|
47
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
require File.expand_path(__FILE__).sub(%r(/test/.*), '/test/test_helper.rb')
|
2
|
+
require File.expand_path(__FILE__).sub(%r(.*/test/), '').sub(/test_(.*)\.rb/,'\1')
|
3
|
+
|
4
|
+
class TestNamedArrayEntity < Test::Unit::TestCase
|
5
|
+
def setup
|
6
|
+
m = Module.new do
|
7
|
+
extend Entity
|
8
|
+
self.format = "SomeEntity"
|
9
|
+
|
10
|
+
property :prop do
|
11
|
+
"PROP: #{self}"
|
12
|
+
end
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def test_true
|
17
|
+
a = NamedArray.setup(["a", "b"], %w(SomeEntity Other))
|
18
|
+
assert a["SomeEntity"].respond_to?(:prop)
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
@@ -40,6 +40,7 @@ row2 a a id3
|
|
40
40
|
end
|
41
41
|
end
|
42
42
|
|
43
|
+
assert_include tsv.fields, "ValueA"
|
43
44
|
assert_include tsv.keys, 'row1'
|
44
45
|
assert_include tsv.keys, 'row2'
|
45
46
|
end
|
@@ -67,6 +68,7 @@ row2 a a id3
|
|
67
68
|
end
|
68
69
|
refute Open.exists?(tk)
|
69
70
|
assert Open.exists?(data.persistence_path)
|
71
|
+
refute tsv.fields.nil?
|
70
72
|
end
|
71
73
|
|
72
74
|
assert_include tsv.keys, 'row1'
|
@@ -142,5 +144,23 @@ row2 a a id3
|
|
142
144
|
assert_include tsv.keys, 'row1'
|
143
145
|
assert_include tsv.keys, 'row2'
|
144
146
|
end
|
147
|
+
|
148
|
+
def test_tsv_open_persist
|
149
|
+
content =<<-'EOF'
|
150
|
+
#Id ValueA ValueB OtherID
|
151
|
+
row1 a|aa|aaa b Id1|Id2
|
152
|
+
row2 A B Id3
|
153
|
+
row2 a a id3
|
154
|
+
EOF
|
155
|
+
|
156
|
+
|
157
|
+
tsv = nil
|
158
|
+
TmpFile.with_file(content) do |filename|
|
159
|
+
tsv = TSV.open(filename, sep: /\s+/, type: :double, persist: true, merge: true)
|
160
|
+
assert Array === tsv.fields
|
161
|
+
tsv = TSV.open(filename, sep: /\s+/, type: :double, persist: true, merge: true)
|
162
|
+
assert Array === tsv.fields
|
163
|
+
end
|
164
|
+
end
|
145
165
|
end
|
146
166
|
|
@@ -39,5 +39,25 @@ class TestTSVAdapter < Test::Unit::TestCase
|
|
39
39
|
assert_equal [1, 2, 3], tsv["a"]
|
40
40
|
assert_equal [1, 2, 3], Marshal.load(tsv.orig_get("a"))
|
41
41
|
end
|
42
|
+
|
43
|
+
def test_persist_annotations
|
44
|
+
content =<<-'EOF'
|
45
|
+
#: :sep=/\s+/#:type=:double#:merge=:concat
|
46
|
+
#Id ValueA ValueB OtherID
|
47
|
+
row1 a|aa|aaa b Id1|Id2
|
48
|
+
row2 A B Id3
|
49
|
+
row2 a a id3
|
50
|
+
EOF
|
51
|
+
|
52
|
+
TmpFile.with_file(content) do |tsv_file|
|
53
|
+
pid = Process.fork do
|
54
|
+
tsv = TSV.open tsv_file, persist: true
|
55
|
+
end
|
56
|
+
Process.waitpid pid
|
57
|
+
tsv = TSV.open tsv_file, persist: true
|
58
|
+
refute tsv.fields.nil?
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
42
62
|
end
|
43
63
|
|