datafarming 2.3.2 → 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,186 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ RubyVM::YJIT.enable if defined?(RubyVM::YJIT) && RubyVM::YJIT.respond_to?(:enable)
4
+
5
+ require 'open3'
6
+ require 'optparse'
7
+ require 'yaml'
8
+
9
+ def is_f?(s)
10
+ !!Float(s)
11
+ rescue StandardError
12
+ false
13
+ end
14
+
15
+ def is_i?(s)
16
+ /\A[-+]?\d+\z/ === s
17
+ end
18
+
19
+ def convert(s)
20
+ return s.to_i if is_i? s
21
+ return s.to_f if is_f? s
22
+ s
23
+ end
24
+
25
+ PROG = File.basename($PROGRAM_NAME).freeze
26
+
27
+ HELP_MSG = [
28
+ '',
29
+ 'Generate valid YAML to perform designed experiments with your model.',
30
+ 'This program works with models that use YAML streaming to assign',
31
+ 'input values at run-time.',
32
+ '',
33
+ "Usage: #{PROG} [options]"
34
+ ].freeze
35
+
36
+ EXAMPLES = [
37
+ '',
38
+ 'PREFACE:',
39
+ '',
40
+ " python mymodel.py < BASELINE.yml > MODELOUT.csv",
41
+ '',
42
+ " If 'mymodel.py' is a model written in Python that uses YAML stream",
43
+ " input to control run-time parameterization, and BASELINE.yml is a",
44
+ " valid and complete input specification, this command runs the model",
45
+ " getting input from BASELINE.yml and directing the STDOUT output to",
46
+ ' MODELOUT.csv. Capitalized names, such as BASELINE and MODELOUT, can',
47
+ ' be replaced throughout the following with any valid name you choose.',
48
+ '',
49
+ 'EXAMPLES:',
50
+ '',
51
+ " #{PROG} -b BASELINE.yml -i DESIGN.csv -o GENERATED.yml",
52
+ '',
53
+ " Use 'DESIGN.csv', which contains a designed experiment in the form",
54
+ " of a CSV file with a header line (as created by 'generate_design.rb')",
55
+ " to modify BASELINE.yml and write the resulting set of YAML documents",
56
+ " (one per design point) to the output file GENERATED.yml. If the '-o'",
57
+ " option is omitted, results go to STDOUT, which by default shows on",
58
+ " the terminal but can be redirected or piped to other destinations.",
59
+ '',
60
+ " The entire designed experiment could then be run with the command:",
61
+ '',
62
+ " python mymodel.py < GENERATED.yml",
63
+ '',
64
+ " Alternatively, this two-stage operation can be combined using piping",
65
+ " for the standard output stream:",
66
+ '',
67
+ " #{PROG} -b BASELINE.yml -i DESIGN.csv | python mymodel.py",
68
+ '',
69
+ " You can use the '-r' option rather than external piping:",
70
+ '',
71
+ " #{PROG} -b BASELINE.yml -i DESIGN.csv -r 'python mymodel.py'",
72
+ '',
73
+ " but we don't recommend doing so - it can be significantly slower than",
74
+ ' using explicit piping due to the overhead of starting a new shell',
75
+ ' environment for every design point.',
76
+ '',
77
+ '',
78
+ ].freeze
79
+
80
+ def prog_help(opts)
81
+ puts opts
82
+ puts
83
+ exit
84
+ end
85
+
86
+ def examples
87
+ puts EXAMPLES.join("\n")
88
+ exit
89
+ end
90
+
91
+ options = {
92
+ :replications => 1,
93
+ :basecase => nil,
94
+ :designfile => nil,
95
+ :outfile => nil,
96
+ :run_model => nil,
97
+ }
98
+
99
+ (ARGV << '-h') if ARGV.empty?
100
+
101
+ OptionParser.new do |opts|
102
+ opts.separator ""
103
+ opts.banner = HELP_MSG.join("\n")
104
+
105
+ opts.separator "Required options:"
106
+ opts.on(
107
+ "-b BASECASE", "--basecase",
108
+ "The name of your template YAML file.",
109
+ "BASECASE must contain settings for all",
110
+ "of your model's parametric inputs.",
111
+ ) { |fname| options[:basecase] = YAML.load_file(fname) }
112
+ opts.on('-i DESIGNFILE', '--designfile',
113
+ "The name of your design file.",
114
+ "DESIGNFILE *must* contain column headers,",
115
+ "which are used to identify and update",
116
+ "specific BASECASE parameter values.",
117
+ "Format: column headers, followed by",
118
+ "one line per design point. Factor values",
119
+ "can be separated by blanks or commas."
120
+ ) do |fname|
121
+ options[:designfile] = File.readlines(fname).map do |line|
122
+ line.strip.split(/[\s,]+/)
123
+ end
124
+ end
125
+ opts.separator ""
126
+ opts.separator "Elective options:"
127
+ opts.on('-r MODEL_CMD', '--run_model',
128
+ 'The command to launch your model.',
129
+ 'This MUST be in single quotes if the',
130
+ 'invocation contains whitespace!',
131
+ " Example: -r 'python mymodel.py'",
132
+ "Use of direct piping to your model",
133
+ "is strongly recommended."
134
+ )
135
+ opts.on(
136
+ '-o', '--outfile YAMLOUTFILE',
137
+ "Write YAML output to the specified file.",
138
+ "(default: STDOUT)"
139
+ )
140
+ opts.on('-n NUMBER', '--replications',
141
+ "The number of replications (default: 1).",
142
+ "Note: all design points are completed",
143
+ "before starting the next replication to",
144
+ "minimize the risk of omitting design",
145
+ "points if the run gets interrupted."
146
+ ) { |num| num.to_i }
147
+ opts.on('-x', '--example', "Provide examples of use.") do
148
+ examples
149
+ exit
150
+ end
151
+ opts.on_tail('-h', '-?', '--help',
152
+ 'Display this help message.'
153
+ ) { prog_help(opts) }
154
+ end.parse!(into: options)
155
+
156
+ fail("Missing required '--designfile' specification!") unless options[:designfile]
157
+ fail("Missing required '--basecase' YAML specification!") unless options[:basecase]
158
+ fail("Invalid arguments: #{ARGV.join(' ')}") if ARGV.size > 0
159
+ fail('Missing header line?') if options[:designfile][0].any? { |item| is_f?(item) || is_i?(item) }
160
+
161
+ options[:labels] = options[:designfile].shift # .map(&options[:label_proc])
162
+
163
+ def write_to(io, options)
164
+ options[:replications].times do
165
+ options[:designfile].each do |design_pt|
166
+ options[:labels].each_with_index do |key, i|
167
+ options[:basecase][key] = convert(design_pt[i])
168
+ end
169
+ if options[:run_model]
170
+ Open3.pipeline_rw([options[:run_model]]) do |in_strm, out_strm, wait|
171
+ in_strm.puts options[:basecase].to_yaml
172
+ in_strm.close
173
+ io.puts out_strm.read
174
+ end
175
+ else
176
+ io.puts options[:basecase].to_yaml
177
+ end
178
+ end
179
+ end
180
+ end
181
+
182
+ if options[:outfile]
183
+ File.open(options[:outfile], "w") { |file| write_to(file, options) }
184
+ else
185
+ write_to(STDOUT, options)
186
+ end
@@ -1,85 +1,144 @@
1
1
  #!/usr/bin/env ruby -w
2
2
 
3
- require 'colorize'
4
-
5
- String.disable_colorization false
3
+ RubyVM::YJIT.enable if defined?(RubyVM::YJIT) && RubyVM::YJIT.respond_to?(:enable)
6
4
 
7
5
  require 'optparse'
8
- require 'datafarming/error_handling'
9
6
 
10
- help_msg = [
11
- 'Run control to apply a designed experiment to a model with replication.', '',
12
- 'This script assumes that the model uses command-line arguments to',
13
- 'set factor values at run-time.', '',
14
- 'Syntax:',
15
- "\n\t#{ErrorHandling.prog_name} [OPTIONS] ".yellow +
16
- "'CMD' DOE_FILE #REPS OUTPUT_FILE".yellow, '',
17
- "Arguments in square brackets are optional. A vertical bar '|'",
18
- 'indicates valid alternatives for invoking the option. Prefix',
19
- 'the command with "' + 'ruby'.yellow +
20
- '" if it is not on your PATH.', '',
21
- ' --help | -h | -? | ?'.green,
22
- "\tProduce this help message.",
23
- ' --print | -p'.green,
24
- "\tPrint generated commands rather than executing them,",
25
- "\tuseful for debugging.",
26
- ' --destructive | -d'.green,
27
- "\tOverwrite any prior contents in the output file. Default",
28
- "\tbehavior is to append new results to an existing output file.", '',
29
- 'Required arguments are:', '',
30
- " 'CMD'".green,
31
- "\tThe command to run the model. " +
32
- 'MUST be placed in single quotes'.red,
33
- "\tif the command contains any white space or special characters.",
34
- "\tExample: " + "'java MyModel.jar'".blue,
35
- ' DOE_FILE'.green,
36
- "\tThe name of a text file containing the experimental design",
37
- "\tto be used. The design file should have one line per design",
38
- "\tpoint with factor settings separated by white space. Factor",
39
- "\tsettings must be provided in the order expected by the model.",
40
- ' #REPS'.green,
41
- "\tAn integer specifying the number of times each design point",
42
- "\tshould be replicated. All design points are completed before",
43
- "\tmoving to the next replication to minimize the risk of missing",
44
- "\tdesign points if the run gets interrupted for any reason.",
45
- ' OUTPUT_FILE'.green,
46
- "\tThe name of a text file to which all output will be written."
47
- ]
7
+ def print_it(design:, reps:, script_dest:, cmd_dest:)
8
+ append = cmd_dest ? " >> #{cmd_dest}" : ""
9
+ reps.times do
10
+ design.each { |line| script_dest.puts(line + append) }
11
+ end
12
+ end
13
+
14
+ def run_it(design:, reps:, cmd_dest:)
15
+ reps.times do
16
+ design.each do |cmd|
17
+ result = `#{cmd}`
18
+ cmd_dest.puts(result) if result =~ /\S/
19
+ end
20
+ end
21
+ end
22
+
23
+ HELP_MSG = [
24
+ 'Run control to apply a designed experiment to a model with replication.',
25
+ 'Use this program to run models that use command-line arguments to',
26
+ 'assign factor values at run-time and write their results to STDOUT.',
27
+ '',
28
+ "Usage: #{File.basename($PROGRAM_NAME)} [options]"
29
+ ].freeze
30
+
31
+ def prog_help(opts)
32
+ puts opts
33
+ puts
34
+ exit
35
+ end
36
+
37
+ options = {
38
+ :script => nil, # default is to run rather than print
39
+ :script_wanted => false,
40
+ :destructive => false, # default is non-destructive for output file
41
+ :replications => 1,
42
+ :infile => nil,
43
+ :outfile => nil,
44
+ :run_model => nil,
45
+ }
46
+
47
+ (ARGV << '-h') if ARGV.empty?
48
48
 
49
- print_cmds = false # default is to run rather than print
50
- destructive = false # default is non-destructive for output file
51
49
  OptionParser.new do |opts|
52
- opts.banner = "Usage: #{$PROGRAM_NAME} [-h|--help] [filenames...[]"
53
- opts.on('-h', '-?', '--help') { ErrorHandling.clean_abort help_msg }
54
- opts.on('-p', '--print') { print_cmds = true }
55
- opts.on('-d', '--destructive') { destructive = true }
56
- end.parse!
50
+ opts.separator ""
51
+ opts.banner = HELP_MSG.join("\n")
52
+
53
+ opts.separator "Required options:"
54
+ opts.on('-r MODEL_CMD', '--run_model',
55
+ 'The command to launch your model.',
56
+ 'This MUST be in single quotes if the',
57
+ 'invocation contains whitespace!',
58
+ " Example: -r 'python mymodel.py'"
59
+ )
60
+ opts.on('-i DESIGNFILE', '--infile',
61
+ "The name of your design file.",
62
+ "The design cannot contain column headers.",
63
+ "Format: one line per design point, factor",
64
+ "levels separated by blanks or commas."
65
+ )
66
+ opts.separator ""
67
+ opts.separator "Elective options:"
68
+ opts.on(
69
+ '-o', '--outfile OUTFILE',
70
+ "File name for execution output.",
71
+ "(default: STDOUT)"
72
+ )
73
+ opts.on('-n NUMBER', '--replications',
74
+ "The number of replications (default: 1).",
75
+ "Note: all design points are completed",
76
+ "before starting the next replication to",
77
+ "minimize the risk of omitting design",
78
+ "points if the run gets interrupted."
79
+ ) { |num| num.to_i }
80
+ opts.on('-s', '--script [SCRIPTNAME]',
81
+ "Do NOT execute the generated commands,",
82
+ "output them as a script instead. Commands",
83
+ "are written to SCRIPTNAME (if provided)",
84
+ "or sent to STDOUT.",
85
+ ) do |s_name|
86
+ options[:script_wanted] = true
87
+ options[:script] = s_name
88
+ end
89
+ opts.on('-d', '--destructive',
90
+ "Overwrite the output file.",
91
+ "(default: append to file)"
92
+ ) { true }
93
+ opts.on_tail('-h', '-?', '--help',
94
+ 'Display this help message.'
95
+ ) { prog_help(opts) }
96
+ end.parse!(into: options)
57
97
 
58
- ErrorHandling.clean_abort help_msg if ARGV[0] == '?' || ARGV.length != 4
98
+ fail("Missing required '--run_model' specification!") unless options[:run_model]
99
+ fail("Missing required '--infile' specification!") unless options[:infile]
100
+ fail("Invalid arguments: #{ARGV.join(' ')}") if ARGV.size > 0
59
101
 
60
- begin
61
- # What shall we run today?
62
- cmd = ARGV.shift
63
- # Suck in all the design points from the file specified as the next
64
- # argument, strip the whitespace, and put the results in an array
65
- design_pts = File.readlines(ARGV.shift).map(&:strip!)
66
- # How many times do we want to do this?
67
- reps = ARGV.shift.to_i
68
- # Where do the results go?
69
- output_file_name = ARGV.shift
70
- File.delete(output_file_name) if destructive && File.exist?(output_file_name)
71
- reps.times do
72
- design_pts.each do |design_pt|
73
- exe_line = "#{cmd} #{design_pt} >> #{output_file_name}"
74
- if print_cmds
75
- puts exe_line
76
- else
77
- result = `#{exe_line}`
78
- STDERR.puts result if result =~ /\S/
79
- end
102
+ if options[:destructive] && options[:script_wanted]
103
+ STDERR.puts
104
+ STDERR.puts " =================================================================="
105
+ STDERR.puts " The '--destructive' option is ignored when generating a '--script'"
106
+ STDERR.puts " with a specified '--outfile'. If that's something you really want"
107
+ STDERR.puts " to do, edit the generated script file and change '>>' to '>' on"
108
+ STDERR.puts " the first line."
109
+ STDERR.puts " =================================================================="
110
+ STDERR.puts
111
+ end
112
+
113
+ design_pts = File.readlines(options[:infile]).map do |line|
114
+ "#{options[:run_model]} #{line.strip.gsub(',', ' ')}"
115
+ end
116
+
117
+ if options[:script_wanted]
118
+ if options[:script].nil?
119
+ print_it(
120
+ design: design_pts,
121
+ reps: options[:replications],
122
+ script_dest: STDOUT,
123
+ cmd_dest: options[:outfile]
124
+ )
125
+ else
126
+ File.open(options[:script], "w") do |f|
127
+ print_it(
128
+ design: design_pts,
129
+ reps: options[:replications],
130
+ script_dest: f,
131
+ cmd_dest: options[:outfile]
132
+ )
133
+ end
134
+ end
135
+ else
136
+ if options[:outfile].nil?
137
+ run_it(design: design_pts, reps: options[:replications], cmd_dest: STDOUT)
138
+ else
139
+ mode = options[:destructive] ? "w" : "a"
140
+ File.open(options[:outfile], mode) do |f|
141
+ run_it(design: design_pts, reps: options[:replications], cmd_dest: f)
80
142
  end
81
143
  end
82
- rescue StandardError => e
83
- ErrorHandling.message [e.message.red]
84
- ErrorHandling.clean_abort help_msg
85
144
  end
@@ -247,8 +247,8 @@ module FBD
247
247
  ]
248
248
  ),
249
249
  50 => DesignSet.new(
250
- nyq: 16843, freqs: [
251
- [1,27,32,33,42,71,79,124,160,173,180,259,369,442,469,499,636,703,798,924,1109,1192,1389,1403,1645,1696,1767,1953,2218,2335,2584,2609,2785,2862,3022,3371,3510,3642,4016,4123,4288,4864,5134,5459,5593,5880,6185,6480,7471,7662]
250
+ nyq: 16229, freqs: [
251
+ [1, 5, 17, 22, 25, 36, 68, 127, 134, 167, 182, 242, 320, 349, 358, 403, 600, 649, 790, 858, 914, 995, 1196, 1326, 1408, 1473, 1769, 1770, 1996, 2075, 2313, 2732, 2942, 3289, 3341, 3460, 3552, 3692, 3786, 4031, 4244, 4473, 4828, 4841, 5218, 5522, 5936, 6680, 7262, 7725]
252
252
  ]
253
253
  ),
254
254
  51 => DesignSet.new(
@@ -297,8 +297,8 @@ module FBD
297
297
  ]
298
298
  ),
299
299
  60 => DesignSet.new(
300
- nyq: 27967, freqs: [
301
- [1,30,35,42,57,63,67,81,116,199,216,270,355,410,458,537,662,775,906,1001,1002,1178,1225,1368,1443,1456,1584,1860,1966,2267,2527,2739,2923,3014,3251,3567,3917,4201,4221,4779,5042,5287,5574,5935,6134,6212,6595,6891,6917,7322,7709,8194,8705,8809,8866,9531,10015,10537,11411,11830]
300
+ nyq: 27751, freqs: [
301
+ [1, 15, 20, 49, 58, 75, 76, 86, 196, 204, 293, 360, 373, 405, 473, 531, 615, 696, 807, 943, 997, 1128, 1180, 1359, 1558, 1623, 1740, 1764, 2225, 2266, 2493, 2734, 2857, 3083, 3089, 3741, 3992, 4285, 4444, 4756, 4861, 5318, 5381, 5896, 5900, 6291, 6732, 7488, 7745, 7975, 8187, 8563, 9148, 9479, 9804, 9944, 10538, 10589, 11106, 11627]
302
302
  ]
303
303
  ),
304
304
  61 => DesignSet.new(
@@ -347,9 +347,9 @@ module FBD
347
347
  ]
348
348
  ),
349
349
  70 => DesignSet.new(
350
- nyq: 43313, freqs: [
351
- [1,22,25,36,42,64,82,91,95,203,257,265,290,391,506,562,664,702,902,1025,1105,1120,1311,1455,1564,1615,1649,1929,2139,2140,2401,2483,3004,3240,3582,3692,3919,4313,4435,4465,4510,4968,5527,5707,6146,6578,6962,7363,7681,7838,8415,8778,9140,9395,9719,10440,10902,11574,11974,12773,13764,13780,14169,15081,15498,16087,16932,17649,18166,18319]
352
- ]
350
+ nyq: 42859, freqs: [
351
+ [1, 22, 40, 72, 86, 91, 106, 149, 189, 197, 271, 330, 423, 475, 524, 577, 666, 741, 840, 871, 1006, 1216, 1252, 1297, 1549, 1999, 2000, 2120, 2137, 2256, 2699, 3053, 3239, 3490, 3518, 3551, 4384, 4689, 4718, 4857, 4943, 5185, 5303, 5896, 6309, 6619, 7119, 7781, 8282, 8392, 8722, 9014, 9026, 9550, 9719, 10434, 10643, 11616, 12253, 12387, 12711, 13931, 14298, 14806, 15820, 16313, 16355, 17337, 17703, 18876]
352
+ ]
353
353
  ),
354
354
  71 => DesignSet.new(
355
355
  nyq: 44203, freqs: [
@@ -397,8 +397,8 @@ module FBD
397
397
  ]
398
398
  ),
399
399
  80 => DesignSet.new(
400
- nyq: 66377, freqs: [
401
- [1,27,38,48,51,56,83,143,173,189,211,325,356,368,465,553,614,634,761,864,1062,1121,1251,1314,1506,1686,1851,2083,2087,2406,2657,2699,2750,3217,3451,3666,3699,4159,4382,4497,4621,5140,5290,5641,5888,6169,6626,7353,7461,7724,8116,8659,8782,9113,9138,10007,10409,10480,11397,12100,12662,13516,14121,14770,14977,15666,16289,16991,17566,17956,18585,19558,20465,21086,21770,22444,23139,23864,24807,25266]
400
+ nyq: 61223, freqs: [
401
+ [1, 10, 13, 14, 46, 68, 76, 107, 117, 155, 246, 303, 346, 418, 458, 523, 541, 745, 787, 997, 1085, 1223, 1325, 1420, 1599, 1787, 1808, 1936, 2142, 2555, 2745, 2870, 2921, 3308, 3651, 3715, 4111, 4475, 4626, 4678, 4996, 5158, 5371, 5670, 5928, 6357, 6809, 7153, 7734, 7951, 7988, 8700, 9416, 9907, 10722, 10798, 11119, 11918, 12749, 12943, 13103, 13386, 14021, 14301, 15404, 15883, 16633, 17006, 17139, 19479, 19496, 19787, 21189, 21974, 22409, 23106, 24288, 24470, 25017, 25522]
402
402
  ]
403
403
  ),
404
404
  81 => DesignSet.new(
@@ -3,13 +3,21 @@
3
3
  # A Scaler object will rescale a design from standard range [-1, 1] to a
4
4
  # range specified by min & max, with the desired number of decimals
5
5
  class Scaler
6
- def initialize(min: -1, max: 1, decimals: nil)
7
- unless decimals.nil? || ((decimals >= 0) && (decimals == decimals.to_i))
6
+ # def initialize(min: -1, max: 1, decimals: nil)
7
+ def initialize(spec)
8
+ unless spec[:decimals].nil? ||
9
+ ((spec[:decimals] >= 0) && (spec[:decimals] == spec[:decimals].to_i))
8
10
  fail "Invalid decimals argument: #{decimals}. Must be a non-negative integer"
9
11
  end
10
- @range = (max - min).to_r / 2
11
- @mid = min + @range
12
- @scale_factor = (decimals.nil? || decimals.zero?) ? decimals : 10r**decimals
12
+ @spec = spec
13
+ if @spec[:discrete]
14
+ @scale_factor = 1
15
+ else
16
+ @range = (spec[:max] - spec[:min]).to_r / 2
17
+ @mid = spec[:min] + @range
18
+ @scale_factor = (spec[:decimals].nil? || spec[:decimals].zero?) ?
19
+ spec[:decimals] : 10r**spec[:decimals]
20
+ end
13
21
  end
14
22
 
15
23
  def scale(value)
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module DataFarming
4
- VERSION = "2.3.2"
4
+ VERSION = "3.0.0"
5
5
  end