datafarming 2.3.2 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/LICENSE.md +1 -1
- data/README.md +13 -9
- data/datafarming.gemspec +2 -2
- data/exe/datafarming.rb +18 -14
- data/exe/generate_design.rb +195 -70
- data/exe/mser_nolbm.rb +8 -6
- data/exe/mser_olbm.rb +2 -0
- data/exe/run_yaml_design.rb +186 -0
- data/exe/rundesign_general.rb +132 -73
- data/lib/datafarming/freq_sets.rb +9 -9
- data/lib/datafarming/scale_design.rb +13 -5
- data/lib/datafarming/version.rb +1 -1
- metadata +143 -132
- data/exe/mser.rb +0 -71
metadata
CHANGED
|
@@ -1,155 +1,166 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: datafarming
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version:
|
|
4
|
+
version: 3.0.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
|
-
- Paul J Sanchez
|
|
7
|
+
- Paul J Sanchez
|
|
8
8
|
bindir: exe
|
|
9
9
|
cert_chain: []
|
|
10
|
-
date: 2026-
|
|
10
|
+
date: 2026-11-20 00:00:00.000000000 Z
|
|
11
11
|
dependencies:
|
|
12
|
-
- !ruby/object:Gem::Dependency
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
- !ruby/object:Gem::
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
12
|
+
- !ruby/object:Gem::Dependency
|
|
13
|
+
name: fwt
|
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
|
15
|
+
requirements:
|
|
16
|
+
-
|
|
17
|
+
- ~>
|
|
18
|
+
- !ruby/object:Gem::Version
|
|
19
|
+
version: "1.2"
|
|
20
|
+
type: :runtime
|
|
21
|
+
prerelease: false
|
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
23
|
+
requirements:
|
|
24
|
+
-
|
|
25
|
+
- ~>
|
|
26
|
+
- !ruby/object:Gem::Version
|
|
27
|
+
version: "1.2"
|
|
28
|
+
- !ruby/object:Gem::Dependency
|
|
29
|
+
name: colorize
|
|
30
|
+
requirement: !ruby/object:Gem::Requirement
|
|
31
|
+
requirements:
|
|
32
|
+
-
|
|
33
|
+
- ~>
|
|
34
|
+
- !ruby/object:Gem::Version
|
|
35
|
+
version: "1"
|
|
36
|
+
type: :runtime
|
|
37
|
+
prerelease: false
|
|
38
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
39
|
+
requirements:
|
|
40
|
+
-
|
|
41
|
+
- ~>
|
|
42
|
+
- !ruby/object:Gem::Version
|
|
43
|
+
version: "1"
|
|
44
|
+
- !ruby/object:Gem::Dependency
|
|
45
|
+
name: quickstats
|
|
46
|
+
requirement: !ruby/object:Gem::Requirement
|
|
47
|
+
requirements:
|
|
48
|
+
-
|
|
49
|
+
- ~>
|
|
50
|
+
- !ruby/object:Gem::Version
|
|
51
|
+
version: "2"
|
|
52
|
+
type: :runtime
|
|
53
|
+
prerelease: false
|
|
54
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
55
|
+
requirements:
|
|
56
|
+
-
|
|
57
|
+
- ~>
|
|
58
|
+
- !ruby/object:Gem::Version
|
|
59
|
+
version: "2"
|
|
60
|
+
- !ruby/object:Gem::Dependency
|
|
61
|
+
name: optparse
|
|
62
|
+
requirement: !ruby/object:Gem::Requirement
|
|
63
|
+
requirements:
|
|
64
|
+
-
|
|
65
|
+
- ~>
|
|
66
|
+
- !ruby/object:Gem::Version
|
|
67
|
+
version: "0.4"
|
|
68
|
+
type: :runtime
|
|
69
|
+
prerelease: false
|
|
70
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
71
|
+
requirements:
|
|
72
|
+
-
|
|
73
|
+
- ~>
|
|
74
|
+
- !ruby/object:Gem::Version
|
|
75
|
+
version: "0.4"
|
|
76
|
+
- !ruby/object:Gem::Dependency
|
|
77
|
+
name: yaml
|
|
78
|
+
requirement: !ruby/object:Gem::Requirement
|
|
79
|
+
requirements:
|
|
80
|
+
-
|
|
81
|
+
- ~>
|
|
82
|
+
- !ruby/object:Gem::Version
|
|
83
|
+
version: "0.3"
|
|
84
|
+
type: :runtime
|
|
85
|
+
prerelease: false
|
|
86
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
87
|
+
requirements:
|
|
88
|
+
-
|
|
89
|
+
- ~>
|
|
90
|
+
- !ruby/object:Gem::Version
|
|
91
|
+
version: "0.3"
|
|
92
|
+
description: "Ruby scripts for data farming, including pre- andpost-processing, design generation and scaling, and run control."
|
|
93
|
+
email: prof.pjs@icloud.com
|
|
85
94
|
executables:
|
|
86
|
-
- batchrunner.rb
|
|
87
|
-
- blank2csv.rb
|
|
88
|
-
- cat.rb
|
|
89
|
-
- convert_line_endings.rb
|
|
90
|
-
- cross.rb
|
|
91
|
-
- csv2blank.rb
|
|
92
|
-
- datafarming.rb
|
|
93
|
-
- generate_design.rb
|
|
94
|
-
- moving_average.rb
|
|
95
|
-
-
|
|
96
|
-
-
|
|
97
|
-
-
|
|
98
|
-
-
|
|
99
|
-
- rundesign_general.rb
|
|
100
|
-
- stripheaderdups.rb
|
|
101
|
-
- stripheaders.rb
|
|
95
|
+
- batchrunner.rb
|
|
96
|
+
- blank2csv.rb
|
|
97
|
+
- cat.rb
|
|
98
|
+
- convert_line_endings.rb
|
|
99
|
+
- cross.rb
|
|
100
|
+
- csv2blank.rb
|
|
101
|
+
- datafarming.rb
|
|
102
|
+
- generate_design.rb
|
|
103
|
+
- moving_average.rb
|
|
104
|
+
- mser_nolbm.rb
|
|
105
|
+
- mser_olbm.rb
|
|
106
|
+
- pool_files.rb
|
|
107
|
+
- run_yaml_design.rb
|
|
108
|
+
- rundesign_general.rb
|
|
109
|
+
- stripheaderdups.rb
|
|
110
|
+
- stripheaders.rb
|
|
102
111
|
extensions: []
|
|
103
112
|
extra_rdoc_files: []
|
|
104
113
|
files:
|
|
105
|
-
- LICENSE.md
|
|
106
|
-
- README.md
|
|
107
|
-
- datafarming.gemspec
|
|
108
|
-
- exe/batchrunner.rb
|
|
109
|
-
- exe/blank2csv.rb
|
|
110
|
-
- exe/cat.rb
|
|
111
|
-
- exe/convert_line_endings.rb
|
|
112
|
-
- exe/cross.rb
|
|
113
|
-
- exe/csv2blank.rb
|
|
114
|
-
- exe/datafarming.rb
|
|
115
|
-
- exe/generate_design.rb
|
|
116
|
-
- exe/moving_average.rb
|
|
117
|
-
- exe/
|
|
118
|
-
- exe/
|
|
119
|
-
- exe/
|
|
120
|
-
- exe/
|
|
121
|
-
- exe/rundesign_general.rb
|
|
122
|
-
- exe/stripheaderdups.rb
|
|
123
|
-
- exe/stripheaders.rb
|
|
124
|
-
- lib/datafarming.rb
|
|
125
|
-
- lib/datafarming/cross.rb
|
|
126
|
-
- lib/datafarming/error_handling.rb
|
|
127
|
-
- lib/datafarming/freq_sets.rb
|
|
128
|
-
- lib/datafarming/moving_average.rb
|
|
129
|
-
- lib/datafarming/nolh_designs.rb
|
|
130
|
-
- lib/datafarming/res_v_seqs.rb
|
|
131
|
-
- lib/datafarming/scale_design.rb
|
|
132
|
-
- lib/datafarming/screen_freq_sets.rb
|
|
133
|
-
- lib/datafarming/version.rb
|
|
134
|
-
homepage: https://bitbucket.org/paul_j_sanchez/datafarmingrubyscripts
|
|
114
|
+
- LICENSE.md
|
|
115
|
+
- README.md
|
|
116
|
+
- datafarming.gemspec
|
|
117
|
+
- exe/batchrunner.rb
|
|
118
|
+
- exe/blank2csv.rb
|
|
119
|
+
- exe/cat.rb
|
|
120
|
+
- exe/convert_line_endings.rb
|
|
121
|
+
- exe/cross.rb
|
|
122
|
+
- exe/csv2blank.rb
|
|
123
|
+
- exe/datafarming.rb
|
|
124
|
+
- exe/generate_design.rb
|
|
125
|
+
- exe/moving_average.rb
|
|
126
|
+
- exe/mser_nolbm.rb
|
|
127
|
+
- exe/mser_olbm.rb
|
|
128
|
+
- exe/pool_files.rb
|
|
129
|
+
- exe/run_yaml_design.rb
|
|
130
|
+
- exe/rundesign_general.rb
|
|
131
|
+
- exe/stripheaderdups.rb
|
|
132
|
+
- exe/stripheaders.rb
|
|
133
|
+
- lib/datafarming.rb
|
|
134
|
+
- lib/datafarming/cross.rb
|
|
135
|
+
- lib/datafarming/error_handling.rb
|
|
136
|
+
- lib/datafarming/freq_sets.rb
|
|
137
|
+
- lib/datafarming/moving_average.rb
|
|
138
|
+
- lib/datafarming/nolh_designs.rb
|
|
139
|
+
- lib/datafarming/res_v_seqs.rb
|
|
140
|
+
- lib/datafarming/scale_design.rb
|
|
141
|
+
- lib/datafarming/screen_freq_sets.rb
|
|
142
|
+
- lib/datafarming/version.rb
|
|
143
|
+
homepage: "https://bitbucket.org/paul_j_sanchez/datafarmingrubyscripts"
|
|
135
144
|
licenses:
|
|
136
|
-
- MIT
|
|
145
|
+
- MIT
|
|
137
146
|
metadata: {}
|
|
138
147
|
rdoc_options: []
|
|
139
148
|
require_paths:
|
|
140
|
-
- lib
|
|
149
|
+
- lib
|
|
141
150
|
required_ruby_version: !ruby/object:Gem::Requirement
|
|
142
151
|
requirements:
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
152
|
+
-
|
|
153
|
+
- ">="
|
|
154
|
+
- !ruby/object:Gem::Version
|
|
155
|
+
version: "2.6"
|
|
146
156
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
147
157
|
requirements:
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
158
|
+
-
|
|
159
|
+
- ">="
|
|
160
|
+
- !ruby/object:Gem::Version
|
|
161
|
+
version: "0"
|
|
151
162
|
requirements: []
|
|
152
|
-
rubygems_version: 4.0
|
|
163
|
+
rubygems_version: 4.1.0
|
|
153
164
|
specification_version: 4
|
|
154
|
-
summary: Useful
|
|
165
|
+
summary: Useful tools for data farming.
|
|
155
166
|
test_files: []
|
data/exe/mser.rb
DELETED
|
@@ -1,71 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env ruby -w
|
|
2
|
-
|
|
3
|
-
require 'rubygems' if RUBY_VERSION =~ /^1\.8/
|
|
4
|
-
require 'colorize'
|
|
5
|
-
|
|
6
|
-
String.disable_colorization false
|
|
7
|
-
|
|
8
|
-
require 'optparse'
|
|
9
|
-
require 'datafarming/error_handling'
|
|
10
|
-
|
|
11
|
-
begin
|
|
12
|
-
require 'quickstats'
|
|
13
|
-
rescue LoadError
|
|
14
|
-
ErrorHandling.clean_abort [
|
|
15
|
-
"\n\tALERT: quickstats gem is not installed!".red,
|
|
16
|
-
"\tIf you have network connectivity, type:",
|
|
17
|
-
"\n\t\tgem install quickstats\n".yellow,
|
|
18
|
-
"\t(Admin privileges may be required.)\n\n"
|
|
19
|
-
]
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
help_msg = [
|
|
23
|
-
'Calculate MSER truncation statistics for one or more input files.', '',
|
|
24
|
-
'Input files should consist of one column of data per file, with or',
|
|
25
|
-
'without headers. The output consists of one line per input file,',
|
|
26
|
-
'comprised of the MSER-based average of the data, the number of',
|
|
27
|
-
'observations used to calculate that average, and the number of',
|
|
28
|
-
'observations truncated, separated by commas. Output is written',
|
|
29
|
-
'to ' + 'stdout'.blue + ' in CSV format, with headers.', '',
|
|
30
|
-
'Syntax:',
|
|
31
|
-
"\n\t#{ErrorHandling.prog_name} [--help] filenames...".yellow, '',
|
|
32
|
-
"Arguments in square brackets are optional. A vertical bar '|'",
|
|
33
|
-
'indicates valid alternatives for invoking the option. Prefix',
|
|
34
|
-
'the command with "' + 'ruby'.yellow +
|
|
35
|
-
'" if it is not on your PATH.', '',
|
|
36
|
-
' --help | -h | -? | ?'.green,
|
|
37
|
-
"\tProduce this help message.",
|
|
38
|
-
' filenames...'.green,
|
|
39
|
-
"\tThe names of two or more files containing data to be analyzed."
|
|
40
|
-
]
|
|
41
|
-
|
|
42
|
-
OptionParser.new do |opts|
|
|
43
|
-
opts.banner = "Usage: #{$PROGRAM_NAME} [-h|--help] filenames..."
|
|
44
|
-
opts.on('-h', '-?', '--help') { ErrorHandling.clean_abort help_msg }
|
|
45
|
-
end.parse!
|
|
46
|
-
|
|
47
|
-
ErrorHandling.clean_abort help_msg if ARGV.empty? || ARGV[0] == '?'
|
|
48
|
-
|
|
49
|
-
puts 'x-bar,n,trunc'
|
|
50
|
-
|
|
51
|
-
ARGV.each do |fname|
|
|
52
|
-
data = File.readlines(fname)
|
|
53
|
-
data.shift if data[0] =~ /[A-Za-z]/ # strip header if one present
|
|
54
|
-
data.map! { |line| line.chomp.strip.to_f }
|
|
55
|
-
m_stats = QuickStats.new
|
|
56
|
-
warmup = [(data.length * 0.5).to_i, data.length - 10].min
|
|
57
|
-
index = data.length - 1
|
|
58
|
-
while index > (data.length - warmup) && index > 1
|
|
59
|
-
m_stats.new_obs(data[index])
|
|
60
|
-
index -= 1
|
|
61
|
-
end
|
|
62
|
-
best = [m_stats.std_err, m_stats.avg, warmup]
|
|
63
|
-
|
|
64
|
-
while index > -1
|
|
65
|
-
m_stats.new_obs(data[index])
|
|
66
|
-
best = [m_stats.std_err, m_stats.avg, index] if m_stats.std_err <= best[0]
|
|
67
|
-
index -= 1
|
|
68
|
-
end
|
|
69
|
-
|
|
70
|
-
printf "%f,%d,%d\n", best[1], data.length - best[2], best[2]
|
|
71
|
-
end
|